fix(marketplace): improve app detail page content rendering and UX

Fixed multiple issues with app detail page content display and formatting

.github/workflows/docker-release.yml

@@ -0,0 +1,81 @@
+name: Docker Release
+on:
+  release:
+    types: [published]
+  push:
+    tags:
+      - 'docker-rebuild-v*'  # Allow manual Docker rebuilds via tags
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Extract version from release or tag
+        id: get_version
+        run: |
+          if [ "${{ github.event_name }}" == "release" ]; then
+            # Triggered by release event
+            VERSION="${{ github.event.release.tag_name }}"
+            VERSION=${VERSION#v}  # Remove 'v' prefix
+          else
+            # Triggered by docker-rebuild-v* tag
+            VERSION=${GITHUB_REF#refs/tags/docker-rebuild-v}
+          fi
+          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+          echo "Building Docker images for version: $VERSION"
+
+      - name: Extract major and minor versions
+        id: versions
+        run: |
+          VERSION=${{ steps.get_version.outputs.VERSION }}
+          MAJOR=$(echo $VERSION | cut -d. -f1)
+          MINOR=$(echo $VERSION | cut -d. -f1-2)
+          echo "MAJOR=$MAJOR" >> $GITHUB_OUTPUT
+          echo "MINOR=$MINOR" >> $GITHUB_OUTPUT
+          echo "Semantic versions - Major: $MAJOR, Minor: $MINOR"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
+            unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}
+            unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}
+            unclecode/crawl4ai:latest
+          platforms: linux/amd64,linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Summary
+        run: |
+          echo "## 🐳 Docker Release Complete!" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Published Images" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:latest\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Platforms" >> $GITHUB_STEP_SUMMARY
+          echo "- linux/amd64" >> $GITHUB_STEP_SUMMARY
+          echo "- linux/arm64" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 🚀 Pull Command" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY
+          echo "docker pull unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY

.github/workflows/docs/ARCHITECTURE.md

@@ -0,0 +1,917 @@
+# Workflow Architecture Documentation
+
+## Overview
+
+This document describes the technical architecture of the split release pipeline for Crawl4AI.
+
+---
+
+## Architecture Diagram
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│                         Developer                                │
+│                              │                                   │
+│                              ▼                                   │
+│                    git tag v1.2.3                               │
+│                    git push --tags                              │
+└──────────────────────────────┬──────────────────────────────────┘
+                               │
+                               ▼
+┌─────────────────────────────────────────────────────────────────┐
+│                      GitHub Repository                           │
+│                                                                  │
+│  ┌────────────────────────────────────────────────────────┐   │
+│  │                  Tag Event: v1.2.3                      │   │
+│  └────────────────────────────────────────────────────────┘   │
+│                              │                                   │
+│                              ▼                                   │
+│  ┌────────────────────────────────────────────────────────┐   │
+│  │           release.yml (Release Pipeline)               │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 1. Extract Version                            │     │   │
+│  │  │    v1.2.3 → 1.2.3                            │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 2. Validate Version                           │     │   │
+│  │  │    Tag == __version__.py                      │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 3. Build Python Package                       │     │   │
+│  │  │    - Source dist (.tar.gz)                    │     │   │
+│  │  │    - Wheel (.whl)                             │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 4. Upload to PyPI                             │     │   │
+│  │  │    - Authenticate with token                  │     │   │
+│  │  │    - Upload dist/*                            │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 5. Create GitHub Release                      │     │   │
+│  │  │    - Tag: v1.2.3                              │     │   │
+│  │  │    - Body: Install instructions               │     │   │
+│  │  │    - Status: Published                        │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  └────────────────────────────────────────────────────────┘   │
+│                              │                                   │
+│                              ▼                                   │
+│  ┌────────────────────────────────────────────────────────┐   │
+│  │         Release Event: published (v1.2.3)              │   │
+│  └────────────────────────────────────────────────────────┘   │
+│                              │                                   │
+│                              ▼                                   │
+│  ┌────────────────────────────────────────────────────────┐   │
+│  │         docker-release.yml (Docker Pipeline)           │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 1. Extract Version from Release               │     │   │
+│  │  │    github.event.release.tag_name → 1.2.3     │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 2. Parse Semantic Versions                    │     │   │
+│  │  │    1.2.3 → Major: 1, Minor: 1.2              │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 3. Setup Multi-Arch Build                     │     │   │
+│  │  │    - Docker Buildx                            │     │   │
+│  │  │    - QEMU emulation                           │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 4. Authenticate Docker Hub                    │     │   │
+│  │  │    - Username: DOCKER_USERNAME                │     │   │
+│  │  │    - Token: DOCKER_TOKEN                      │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 5. Build Multi-Arch Images                    │     │   │
+│  │  │    ┌────────────────┬────────────────┐       │     │   │
+│  │  │    │  linux/amd64   │  linux/arm64   │       │     │   │
+│  │  │    └────────────────┴────────────────┘       │     │   │
+│  │  │    Cache: GitHub Actions (type=gha)          │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 6. Push to Docker Hub                         │     │   │
+│  │  │    Tags:                                      │     │   │
+│  │  │    - unclecode/crawl4ai:1.2.3                │     │   │
+│  │  │    - unclecode/crawl4ai:1.2                  │     │   │
+│  │  │    - unclecode/crawl4ai:1                    │     │   │
+│  │  │    - unclecode/crawl4ai:latest               │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  └────────────────────────────────────────────────────────┘   │
+└─────────────────────────────────────────────────────────────────┘
+                               │
+                               ▼
+┌─────────────────────────────────────────────────────────────────┐
+│                     External Services                            │
+│                                                                  │
+│  ┌──────────────┐  ┌──────────────┐  ┌──────────────┐         │
+│  │    PyPI      │  │  Docker Hub  │  │   GitHub     │         │
+│  │              │  │              │  │              │         │
+│  │  crawl4ai    │  │ unclecode/   │  │  Releases    │         │
+│  │  1.2.3       │  │ crawl4ai     │  │  v1.2.3      │         │
+│  └──────────────┘  └──────────────┘  └──────────────┘         │
+└─────────────────────────────────────────────────────────────────┘
+```
+
+---
+
+## Component Details
+
+### 1. Release Pipeline (release.yml)
+
+#### Purpose
+Fast publication of Python package and GitHub release.
+
+#### Input
+- **Trigger**: Git tag matching `v*` (excluding `test-v*`)
+- **Example**: `v1.2.3`
+
+#### Processing Stages
+
+##### Stage 1: Version Extraction
+```bash
+Input:  refs/tags/v1.2.3
+Output: VERSION=1.2.3
+```
+
+**Implementation**:
+```bash
+TAG_VERSION=${GITHUB_REF#refs/tags/v}  # Remove 'refs/tags/v' prefix
+echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT
+```
+
+##### Stage 2: Version Validation
+```bash
+Input:  TAG_VERSION=1.2.3
+Check:  crawl4ai/__version__.py contains __version__ = "1.2.3"
+Output: Pass/Fail
+```
+
+**Implementation**:
+```bash
+PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)")
+if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then
+  exit 1
+fi
+```
+
+##### Stage 3: Package Build
+```bash
+Input:  Source code + pyproject.toml
+Output: dist/crawl4ai-1.2.3.tar.gz
+        dist/crawl4ai-1.2.3-py3-none-any.whl
+```
+
+**Implementation**:
+```bash
+python -m build
+# Uses build backend defined in pyproject.toml
+```
+
+##### Stage 4: PyPI Upload
+```bash
+Input:  dist/*.{tar.gz,whl}
+Auth:   PYPI_TOKEN
+Output: Package published to PyPI
+```
+
+**Implementation**:
+```bash
+twine upload dist/*
+# Environment:
+#   TWINE_USERNAME: __token__
+#   TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+```
+
+##### Stage 5: GitHub Release Creation
+```bash
+Input:  Tag: v1.2.3
+        Body: Markdown content
+Output: Published GitHub release
+```
+
+**Implementation**:
+```yaml
+uses: softprops/action-gh-release@v2
+with:
+  tag_name: v1.2.3
+  name: Release v1.2.3
+  body: |
+    Installation instructions and changelog
+  draft: false
+  prerelease: false
+```
+
+#### Output
+- **PyPI Package**: https://pypi.org/project/crawl4ai/1.2.3/
+- **GitHub Release**: Published release on repository
+- **Event**: `release.published` (triggers Docker workflow)
+
+#### Timeline
+```
+0:00 - Tag pushed
+0:01 - Checkout + Python setup
+0:02 - Version validation
+0:03 - Package build
+0:04 - PyPI upload starts
+0:06 - PyPI upload complete
+0:07 - GitHub release created
+0:08 - Workflow complete
+```
+
+---
+
+### 2. Docker Release Pipeline (docker-release.yml)
+
+#### Purpose
+Build and publish multi-architecture Docker images.
+
+#### Inputs
+
+##### Input 1: Release Event (Automatic)
+```yaml
+Event: release.published
+Data:  github.event.release.tag_name = "v1.2.3"
+```
+
+##### Input 2: Docker Rebuild Tag (Manual)
+```yaml
+Tag: docker-rebuild-v1.2.3
+```
+
+#### Processing Stages
+
+##### Stage 1: Version Detection
+```bash
+# From release event:
+VERSION = github.event.release.tag_name.strip("v")
+# Result: "1.2.3"
+
+# From rebuild tag:
+VERSION = GITHUB_REF.replace("refs/tags/docker-rebuild-v", "")
+# Result: "1.2.3"
+```
+
+##### Stage 2: Semantic Version Parsing
+```bash
+Input:  VERSION=1.2.3
+Output: MAJOR=1
+        MINOR=1.2
+        PATCH=3 (implicit)
+```
+
+**Implementation**:
+```bash
+MAJOR=$(echo $VERSION | cut -d. -f1)    # Extract first component
+MINOR=$(echo $VERSION | cut -d. -f1-2)  # Extract first two components
+```
+
+##### Stage 3: Multi-Architecture Setup
+```yaml
+Setup:
+  - Docker Buildx (multi-platform builder)
+  - QEMU (ARM emulation on x86)
+
+Platforms:
+  - linux/amd64 (x86_64)
+  - linux/arm64 (aarch64)
+```
+
+**Architecture**:
+```
+GitHub Runner (linux/amd64)
+  ├─ Buildx Builder
+  │   ├─ Native: Build linux/amd64 image
+  │   └─ QEMU: Emulate ARM to build linux/arm64 image
+  └─ Generate manifest list (points to both images)
+```
+
+##### Stage 4: Docker Hub Authentication
+```bash
+Input:  DOCKER_USERNAME
+        DOCKER_TOKEN
+Output: Authenticated Docker client
+```
+
+##### Stage 5: Build with Cache
+```yaml
+Cache Configuration:
+  cache-from: type=gha           # Read from GitHub Actions cache
+  cache-to: type=gha,mode=max    # Write all layers
+
+Cache Key Components:
+  - Workflow file path
+  - Branch name
+  - Architecture (amd64/arm64)
+```
+
+**Cache Hierarchy**:
+```
+Cache Entry: main/docker-release.yml/linux-amd64
+  ├─ Layer: sha256:abc123... (FROM python:3.12)
+  ├─ Layer: sha256:def456... (RUN apt-get update)
+  ├─ Layer: sha256:ghi789... (COPY requirements.txt)
+  ├─ Layer: sha256:jkl012... (RUN pip install)
+  └─ Layer: sha256:mno345... (COPY . /app)
+
+Cache Hit/Miss Logic:
+  - If layer input unchanged → cache hit → skip build
+  - If layer input changed → cache miss → rebuild + all subsequent layers
+```
+
+##### Stage 6: Tag Generation
+```bash
+Input:  VERSION=1.2.3, MAJOR=1, MINOR=1.2
+
+Output Tags:
+  - unclecode/crawl4ai:1.2.3    (exact version)
+  - unclecode/crawl4ai:1.2      (minor version)
+  - unclecode/crawl4ai:1        (major version)
+  - unclecode/crawl4ai:latest   (latest stable)
+```
+
+**Tag Strategy**:
+- All tags point to same image SHA
+- Users can pin to desired stability level
+- Pushing new version updates `1`, `1.2`, and `latest` automatically
+
+##### Stage 7: Push to Registry
+```bash
+For each tag:
+  For each platform (amd64, arm64):
+    Push image to Docker Hub
+
+Create manifest list:
+  Manifest: unclecode/crawl4ai:1.2.3
+    ├─ linux/amd64: sha256:abc...
+    └─ linux/arm64: sha256:def...
+
+Docker CLI automatically selects correct platform on pull
+```
+
+#### Output
+- **Docker Images**: 4 tags × 2 platforms = 8 image variants + 4 manifests
+- **Docker Hub**: https://hub.docker.com/r/unclecode/crawl4ai/tags
+
+#### Timeline
+
+**Cold Cache (First Build)**:
+```
+0:00 - Release event received
+0:01 - Checkout + Buildx setup
+0:02 - Docker Hub auth
+0:03 - Start build (amd64)
+0:08 - Complete amd64 build
+0:09 - Start build (arm64)
+0:14 - Complete arm64 build
+0:15 - Generate manifests
+0:16 - Push all tags
+0:17 - Workflow complete
+```
+
+**Warm Cache (Code Change Only)**:
+```
+0:00 - Release event received
+0:01 - Checkout + Buildx setup
+0:02 - Docker Hub auth
+0:03 - Start build (amd64) - cache hit for layers 1-4
+0:04 - Complete amd64 build (only layer 5 rebuilt)
+0:05 - Start build (arm64) - cache hit for layers 1-4
+0:06 - Complete arm64 build (only layer 5 rebuilt)
+0:07 - Generate manifests
+0:08 - Push all tags
+0:09 - Workflow complete
+```
+
+---
+
+## Data Flow
+
+### Version Information Flow
+
+```
+Developer
+  │
+  ▼
+crawl4ai/__version__.py
+  __version__ = "1.2.3"
+  │
+  ├─► Git Tag
+  │     v1.2.3
+  │       │
+  │       ▼
+  │     release.yml
+  │       │
+  │       ├─► Validation
+  │       │     ✓ Match
+  │       │
+  │       ├─► PyPI Package
+  │       │     crawl4ai==1.2.3
+  │       │
+  │       └─► GitHub Release
+  │             v1.2.3
+  │               │
+  │               ▼
+  │           docker-release.yml
+  │               │
+  │               └─► Docker Tags
+  │                     1.2.3, 1.2, 1, latest
+  │
+  └─► Package Metadata
+        pyproject.toml
+          version = "1.2.3"
+```
+
+### Secrets Flow
+
+```
+GitHub Secrets (Encrypted at Rest)
+  │
+  ├─► PYPI_TOKEN
+  │     │
+  │     ▼
+  │   release.yml
+  │     │
+  │     ▼
+  │   TWINE_PASSWORD env var (masked in logs)
+  │     │
+  │     ▼
+  │   PyPI API (HTTPS)
+  │
+  ├─► DOCKER_USERNAME
+  │     │
+  │     ▼
+  │   docker-release.yml
+  │     │
+  │     ▼
+  │   docker/login-action (masked in logs)
+  │     │
+  │     ▼
+  │   Docker Hub API (HTTPS)
+  │
+  └─► DOCKER_TOKEN
+        │
+        ▼
+      docker-release.yml
+        │
+        ▼
+      docker/login-action (masked in logs)
+        │
+        ▼
+      Docker Hub API (HTTPS)
+```
+
+### Artifact Flow
+
+```
+Source Code
+  │
+  ├─► release.yml
+  │     │
+  │     ▼
+  │   python -m build
+  │     │
+  │     ├─► crawl4ai-1.2.3.tar.gz
+  │     │     │
+  │     │     ▼
+  │     │   PyPI Storage
+  │     │     │
+  │     │     ▼
+  │     │   pip install crawl4ai
+  │     │
+  │     └─► crawl4ai-1.2.3-py3-none-any.whl
+  │           │
+  │           ▼
+  │         PyPI Storage
+  │           │
+  │           ▼
+  │         pip install crawl4ai
+  │
+  └─► docker-release.yml
+        │
+        ▼
+      docker build
+        │
+        ├─► Image: linux/amd64
+        │     │
+        │     └─► Docker Hub
+        │           unclecode/crawl4ai:1.2.3-amd64
+        │
+        └─► Image: linux/arm64
+              │
+              └─► Docker Hub
+                    unclecode/crawl4ai:1.2.3-arm64
+```
+
+---
+
+## State Machines
+
+### Release Pipeline State Machine
+
+```
+┌─────────┐
+│  START  │
+└────┬────┘
+     │
+     ▼
+┌──────────────┐
+│ Extract      │
+│ Version      │
+└──────┬───────┘
+       │
+       ▼
+┌──────────────┐      ┌─────────┐
+│ Validate     │─────►│ FAILED  │
+│ Version      │ No   │ (Exit 1)│
+└──────┬───────┘      └─────────┘
+       │ Yes
+       ▼
+┌──────────────┐
+│ Build        │
+│ Package      │
+└──────┬───────┘
+       │
+       ▼
+┌──────────────┐      ┌─────────┐
+│ Upload       │─────►│ FAILED  │
+│ to PyPI      │ Error│ (Exit 1)│
+└──────┬───────┘      └─────────┘
+       │ Success
+       ▼
+┌──────────────┐
+│ Create       │
+│ GH Release   │
+└──────┬───────┘
+       │
+       ▼
+┌──────────────┐
+│  SUCCESS     │
+│ (Emit Event) │
+└──────────────┘
+```
+
+### Docker Pipeline State Machine
+
+```
+┌─────────┐
+│  START  │
+│ (Event) │
+└────┬────┘
+     │
+     ▼
+┌──────────────┐
+│ Detect       │
+│ Version      │
+│ Source       │
+└──────┬───────┘
+       │
+       ▼
+┌──────────────┐
+│ Parse        │
+│ Semantic     │
+│ Versions     │
+└──────┬───────┘
+       │
+       ▼
+┌──────────────┐      ┌─────────┐
+│ Authenticate │─────►│ FAILED  │
+│ Docker Hub   │ Error│ (Exit 1)│
+└──────┬───────┘      └─────────┘
+       │ Success
+       ▼
+┌──────────────┐
+│ Build        │
+│ amd64        │
+└──────┬───────┘
+       │
+       ▼
+┌──────────────┐      ┌─────────┐
+│ Build        │─────►│ FAILED  │
+│ arm64        │ Error│ (Exit 1)│
+└──────┬───────┘      └─────────┘
+       │ Success
+       ▼
+┌──────────────┐
+│ Push All     │
+│ Tags         │
+└──────┬───────┘
+       │
+       ▼
+┌──────────────┐
+│  SUCCESS     │
+└──────────────┘
+```
+
+---
+
+## Security Architecture
+
+### Threat Model
+
+#### Threats Mitigated
+
+1. **Secret Exposure**
+   - Mitigation: GitHub Actions secret masking
+   - Evidence: Secrets never appear in logs
+
+2. **Unauthorized Package Upload**
+   - Mitigation: Scoped PyPI tokens
+   - Evidence: Token limited to `crawl4ai` project
+
+3. **Man-in-the-Middle**
+   - Mitigation: HTTPS for all API calls
+   - Evidence: PyPI, Docker Hub, GitHub all use TLS
+
+4. **Supply Chain Tampering**
+   - Mitigation: Immutable artifacts, content checksums
+   - Evidence: PyPI stores SHA256, Docker uses content-addressable storage
+
+#### Trust Boundaries
+
+```
+┌─────────────────────────────────────────┐
+│         Trusted Zone                     │
+│  ┌────────────────────────────────┐    │
+│  │  GitHub Actions Runner         │    │
+│  │  - Ephemeral VM                │    │
+│  │  - Isolated environment        │    │
+│  │  - Access to secrets           │    │
+│  └────────────────────────────────┘    │
+│                │                         │
+│                │ HTTPS (TLS 1.2+)       │
+│                ▼                         │
+└─────────────────────────────────────────┘
+                 │
+    ┌────────────┼────────────┐
+    │            │            │
+    ▼            ▼            ▼
+┌────────┐  ┌─────────┐  ┌──────────┐
+│  PyPI  │  │  Docker │  │  GitHub  │
+│  API   │  │  Hub    │  │  API     │
+└────────┘  └─────────┘  └──────────┘
+ External     External     External
+  Service      Service      Service
+```
+
+### Secret Management
+
+#### Secret Lifecycle
+
+```
+Creation (Developer)
+  │
+  ├─► PyPI: Create API token (scoped to project)
+  ├─► Docker Hub: Create access token (read/write)
+  │
+  ▼
+Storage (GitHub)
+  │
+  ├─► Encrypted at rest (AES-256)
+  ├─► Access controlled (repo-scoped)
+  │
+  ▼
+Usage (Workflow)
+  │
+  ├─► Injected as env vars
+  ├─► Masked in logs (GitHub redacts on output)
+  ├─► Never persisted to disk (in-memory only)
+  │
+  ▼
+Transmission (API Call)
+  │
+  ├─► HTTPS only
+  ├─► TLS 1.2+ with strong ciphers
+  │
+  ▼
+Rotation (Manual)
+  │
+  └─► Regenerate on PyPI/Docker Hub
+      Update GitHub secret
+```
+
+---
+
+## Performance Characteristics
+
+### Release Pipeline Performance
+
+| Metric | Value | Notes |
+|--------|-------|-------|
+| Cold start | ~2-3 min | First run on new runner |
+| Warm start | ~2-3 min | Minimal caching benefit |
+| PyPI upload | ~30-60 sec | Network-bound |
+| Package build | ~30 sec | CPU-bound |
+| Parallelization | None | Sequential by design |
+
+### Docker Pipeline Performance
+
+| Metric | Cold Cache | Warm Cache (code) | Warm Cache (deps) |
+|--------|-----------|-------------------|-------------------|
+| Total time | 10-15 min | 1-2 min | 3-5 min |
+| amd64 build | 5-7 min | 30-60 sec | 1-2 min |
+| arm64 build | 5-7 min | 30-60 sec | 1-2 min |
+| Push time | 1-2 min | 30 sec | 30 sec |
+| Cache hit rate | 0% | 85% | 60% |
+
+### Cache Performance Model
+
+```python
+def estimate_build_time(changes):
+    base_time = 60  # seconds (setup + push)
+
+    if "Dockerfile" in changes:
+        return base_time + (10 * 60)  # Full rebuild: ~11 min
+    elif "requirements.txt" in changes:
+        return base_time + (3 * 60)   # Deps rebuild: ~4 min
+    elif any(f.endswith(".py") for f in changes):
+        return base_time + 60          # Code only: ~2 min
+    else:
+        return base_time               # No changes: ~1 min
+```
+
+---
+
+## Scalability Considerations
+
+### Current Limits
+
+| Resource | Limit | Impact |
+|----------|-------|--------|
+| Workflow concurrency | 20 (default) | Max 20 releases in parallel |
+| Artifact storage | 500 MB/artifact | PyPI packages small (<10 MB) |
+| Cache storage | 10 GB/repo | Docker layers fit comfortably |
+| Workflow run time | 6 hours | Plenty of headroom |
+
+### Scaling Strategies
+
+#### Horizontal Scaling (Multiple Repos)
+```
+crawl4ai (main)
+  ├─ release.yml
+  └─ docker-release.yml
+
+crawl4ai-plugins (separate)
+  ├─ release.yml
+  └─ docker-release.yml
+
+Each repo has independent:
+  - Secrets
+  - Cache (10 GB each)
+  - Concurrency limits (20 each)
+```
+
+#### Vertical Scaling (Larger Runners)
+```yaml
+jobs:
+  docker:
+    runs-on: ubuntu-latest-8-cores  # GitHub-hosted larger runner
+    # 4x faster builds for CPU-bound layers
+```
+
+---
+
+## Disaster Recovery
+
+### Failure Scenarios
+
+#### Scenario 1: Release Pipeline Fails
+
+**Failure Point**: PyPI upload fails (network error)
+
+**State**:
+- ✓ Version validated
+- ✓ Package built
+- ✗ PyPI upload
+- ✗ GitHub release
+
+**Recovery**:
+```bash
+# Manual upload
+twine upload dist/*
+
+# Retry workflow (re-run from GitHub Actions UI)
+```
+
+**Prevention**: Add retry logic to PyPI upload
+
+#### Scenario 2: Docker Pipeline Fails
+
+**Failure Point**: ARM build fails (dependency issue)
+
+**State**:
+- ✓ PyPI published
+- ✓ GitHub release created
+- ✓ amd64 image built
+- ✗ arm64 image build
+
+**Recovery**:
+```bash
+# Fix Dockerfile
+git commit -am "fix: ARM build dependency"
+
+# Trigger rebuild
+git tag docker-rebuild-v1.2.3
+git push origin docker-rebuild-v1.2.3
+```
+
+**Impact**: PyPI package available, only Docker ARM users affected
+
+#### Scenario 3: Partial Release
+
+**Failure Point**: GitHub release creation fails
+
+**State**:
+- ✓ PyPI published
+- ✗ GitHub release
+- ✗ Docker images
+
+**Recovery**:
+```bash
+# Create release manually
+gh release create v1.2.3 \
+  --title "Release v1.2.3" \
+  --notes "..."
+
+# This triggers docker-release.yml automatically
+```
+
+---
+
+## Monitoring and Observability
+
+### Metrics to Track
+
+#### Release Pipeline
+- Success rate (target: >99%)
+- Duration (target: <3 min)
+- PyPI upload time (target: <60 sec)
+
+#### Docker Pipeline
+- Success rate (target: >95%)
+- Duration (target: <15 min cold, <2 min warm)
+- Cache hit rate (target: >80% for code changes)
+
+### Alerting
+
+**Critical Alerts**:
+- Release pipeline failure (blocks release)
+- PyPI authentication failure (expired token)
+
+**Warning Alerts**:
+- Docker build >15 min (performance degradation)
+- Cache hit rate <50% (cache issue)
+
+### Logging
+
+**GitHub Actions Logs**:
+- Retention: 90 days
+- Downloadable: Yes
+- Searchable: Limited
+
+**Recommended External Logging**:
+```yaml
+- name: Send logs to external service
+  if: failure()
+  run: |
+    curl -X POST https://logs.example.com/api/v1/logs \
+      -H "Content-Type: application/json" \
+      -d "{\"workflow\": \"${{ github.workflow }}\", \"status\": \"failed\"}"
+```
+
+---
+
+## Future Enhancements
+
+### Planned Improvements
+
+1. **Automated Changelog Generation**
+   - Use conventional commits
+   - Generate CHANGELOG.md automatically
+
+2. **Pre-release Testing**
+   - Test builds on `test-v*` tags
+   - Upload to TestPyPI
+
+3. **Notification System**
+   - Slack/Discord notifications on release
+   - Email on failure
+
+4. **Performance Optimization**
+   - Parallel Docker builds (amd64 + arm64 simultaneously)
+   - Persistent runners for better caching
+
+5. **Enhanced Validation**
+   - Smoke tests after PyPI upload
+   - Container security scanning
+
+---
+
+## References
+
+- [GitHub Actions Architecture](https://docs.github.com/en/actions/learn-github-actions/understanding-github-actions)
+- [Docker Build Cache](https://docs.docker.com/build/cache/)
+- [PyPI API Documentation](https://warehouse.pypa.io/api-reference/)
+
+---
+
+**Last Updated**: 2025-01-21
+**Version**: 2.0

.github/workflows/docs/WORKFLOW_REFERENCE.md

@@ -0,0 +1,287 @@
+# Workflow Quick Reference
+
+## Quick Commands
+
+### Standard Release
+```bash
+# 1. Update version
+vim crawl4ai/__version__.py  # Set to "1.2.3"
+
+# 2. Commit and tag
+git add crawl4ai/__version__.py
+git commit -m "chore: bump version to 1.2.3"
+git tag v1.2.3
+git push origin main
+git push origin v1.2.3
+
+# 3. Monitor
+# - PyPI: ~2-3 minutes
+# - Docker: ~1-15 minutes
+```
+
+### Docker Rebuild Only
+```bash
+git tag docker-rebuild-v1.2.3
+git push origin docker-rebuild-v1.2.3
+```
+
+### Delete Tag (Undo Release)
+```bash
+# Local
+git tag -d v1.2.3
+
+# Remote
+git push --delete origin v1.2.3
+
+# GitHub Release
+gh release delete v1.2.3
+```
+
+---
+
+## Workflow Triggers
+
+### release.yml
+| Event | Pattern | Example |
+|-------|---------|---------|
+| Tag push | `v*` | `v1.2.3` |
+| Excludes | `test-v*` | `test-v1.2.3` |
+
+### docker-release.yml
+| Event | Pattern | Example |
+|-------|---------|---------|
+| Release published | `release.published` | Automatic |
+| Tag push | `docker-rebuild-v*` | `docker-rebuild-v1.2.3` |
+
+---
+
+## Environment Variables
+
+### release.yml
+| Variable | Source | Example |
+|----------|--------|---------|
+| `VERSION` | Git tag | `1.2.3` |
+| `TWINE_USERNAME` | Static | `__token__` |
+| `TWINE_PASSWORD` | Secret | `pypi-Ag...` |
+| `GITHUB_TOKEN` | Auto | `ghp_...` |
+
+### docker-release.yml
+| Variable | Source | Example |
+|----------|--------|---------|
+| `VERSION` | Release/Tag | `1.2.3` |
+| `MAJOR` | Computed | `1` |
+| `MINOR` | Computed | `1.2` |
+| `DOCKER_USERNAME` | Secret | `unclecode` |
+| `DOCKER_TOKEN` | Secret | `dckr_pat_...` |
+
+---
+
+## Docker Tags Generated
+
+| Version | Tags Created |
+|---------|-------------|
+| v1.0.0 | `1.0.0`, `1.0`, `1`, `latest` |
+| v1.1.0 | `1.1.0`, `1.1`, `1`, `latest` |
+| v1.2.3 | `1.2.3`, `1.2`, `1`, `latest` |
+| v2.0.0 | `2.0.0`, `2.0`, `2`, `latest` |
+
+---
+
+## Workflow Outputs
+
+### release.yml
+| Output | Location | Time |
+|--------|----------|------|
+| PyPI Package | https://pypi.org/project/crawl4ai/ | ~2-3 min |
+| GitHub Release | Repository → Releases | ~2-3 min |
+| Workflow Summary | Actions → Run → Summary | Immediate |
+
+### docker-release.yml
+| Output | Location | Time |
+|--------|----------|------|
+| Docker Images | https://hub.docker.com/r/unclecode/crawl4ai | ~1-15 min |
+| Workflow Summary | Actions → Run → Summary | Immediate |
+
+---
+
+## Common Issues
+
+| Issue | Solution |
+|-------|----------|
+| Version mismatch | Update `crawl4ai/__version__.py` to match tag |
+| PyPI 403 Forbidden | Check `PYPI_TOKEN` secret |
+| PyPI 400 File exists | Version already published, increment version |
+| Docker auth failed | Regenerate `DOCKER_TOKEN` |
+| Docker build timeout | Check Dockerfile, review build logs |
+| Cache not working | First build on branch always cold |
+
+---
+
+## Secrets Checklist
+
+- [ ] `PYPI_TOKEN` - PyPI API token (project or account scope)
+- [ ] `DOCKER_USERNAME` - Docker Hub username
+- [ ] `DOCKER_TOKEN` - Docker Hub access token (read/write)
+- [ ] `GITHUB_TOKEN` - Auto-provided (no action needed)
+
+---
+
+## Workflow Dependencies
+
+### release.yml Dependencies
+```yaml
+Python: 3.12
+Actions:
+  - actions/checkout@v4
+  - actions/setup-python@v5
+  - softprops/action-gh-release@v2
+PyPI Packages:
+  - build
+  - twine
+```
+
+### docker-release.yml Dependencies
+```yaml
+Actions:
+  - actions/checkout@v4
+  - docker/setup-buildx-action@v3
+  - docker/login-action@v3
+  - docker/build-push-action@v5
+Docker:
+  - Buildx
+  - QEMU (for multi-arch)
+```
+
+---
+
+## Cache Information
+
+### Type
+- GitHub Actions Cache (`type=gha`)
+
+### Storage
+- **Limit**: 10GB per repository
+- **Retention**: 7 days for unused entries
+- **Cleanup**: Automatic LRU eviction
+
+### Performance
+| Scenario | Cache Hit | Build Time |
+|----------|-----------|------------|
+| First build | 0% | 10-15 min |
+| Code change only | 85% | 1-2 min |
+| Dependency update | 60% | 3-5 min |
+| No changes | 100% | 30-60 sec |
+
+---
+
+## Build Platforms
+
+| Platform | Architecture | Devices |
+|----------|--------------|---------|
+| linux/amd64 | x86_64 | Intel/AMD servers, AWS EC2, GCP |
+| linux/arm64 | aarch64 | Apple Silicon, AWS Graviton, Raspberry Pi |
+
+---
+
+## Version Validation
+
+### Pre-Tag Checklist
+```bash
+# Check current version
+python -c "from crawl4ai.__version__ import __version__; print(__version__)"
+
+# Verify it matches intended tag
+# If tag is v1.2.3, version should be "1.2.3"
+```
+
+### Post-Release Verification
+```bash
+# PyPI
+pip install crawl4ai==1.2.3
+python -c "import crawl4ai; print(crawl4ai.__version__)"
+
+# Docker
+docker pull unclecode/crawl4ai:1.2.3
+docker run unclecode/crawl4ai:1.2.3 python -c "import crawl4ai; print(crawl4ai.__version__)"
+```
+
+---
+
+## Monitoring URLs
+
+| Service | URL |
+|---------|-----|
+| GitHub Actions | `https://github.com/{owner}/{repo}/actions` |
+| PyPI Project | `https://pypi.org/project/crawl4ai/` |
+| Docker Hub | `https://hub.docker.com/r/unclecode/crawl4ai` |
+| GitHub Releases | `https://github.com/{owner}/{repo}/releases` |
+
+---
+
+## Rollback Strategy
+
+### PyPI (Cannot Delete)
+```bash
+# Increment patch version
+git tag v1.2.4
+git push origin v1.2.4
+```
+
+### Docker (Can Overwrite)
+```bash
+# Rebuild with fix
+git tag docker-rebuild-v1.2.3
+git push origin docker-rebuild-v1.2.3
+```
+
+### GitHub Release
+```bash
+# Delete release
+gh release delete v1.2.3
+
+# Delete tag
+git push --delete origin v1.2.3
+```
+
+---
+
+## Status Badge Markdown
+
+```markdown
+[![Release Pipeline](https://github.com/{owner}/{repo}/actions/workflows/release.yml/badge.svg)](https://github.com/{owner}/{repo}/actions/workflows/release.yml)
+
+[![Docker Release](https://github.com/{owner}/{repo}/actions/workflows/docker-release.yml/badge.svg)](https://github.com/{owner}/{repo}/actions/workflows/docker-release.yml)
+```
+
+---
+
+## Timeline Example
+
+```
+0:00 - Push tag v1.2.3
+0:01 - release.yml starts
+0:02 - Version validation passes
+0:03 - Package built
+0:04 - PyPI upload starts
+0:06 - PyPI upload complete ✓
+0:07 - GitHub release created ✓
+0:08 - release.yml complete
+0:08 - docker-release.yml triggered
+0:10 - Docker build starts
+0:12 - amd64 image built (cache hit)
+0:14 - arm64 image built (cache hit)
+0:15 - Images pushed to Docker Hub ✓
+0:16 - docker-release.yml complete
+
+Total: ~16 minutes
+Critical path (PyPI + GitHub): ~8 minutes
+```
+
+---
+
+## Contact
+
+For workflow issues:
+1. Check Actions tab for logs
+2. Review this reference
+3. See [README.md](./README.md) for detailed docs

.github/workflows/release.yml

@@ -10,53 +10,53 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write  # Required for creating releases
-    
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-      
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: '3.12'
-      
+
       - name: Extract version from tag
         id: get_version
         run: |
           TAG_VERSION=${GITHUB_REF#refs/tags/v}
           echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT
           echo "Releasing version: $TAG_VERSION"
-      
+
       - name: Install package dependencies
         run: |
           pip install -e .
-      
+
       - name: Check version consistency
         run: |
           TAG_VERSION=${{ steps.get_version.outputs.VERSION }}
           PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)")
-          
+
           echo "Tag version: $TAG_VERSION"
           echo "Package version: $PACKAGE_VERSION"
-          
+
           if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then
             echo "❌ Version mismatch! Tag: $TAG_VERSION, Package: $PACKAGE_VERSION"
             echo "Please update crawl4ai/__version__.py to match the tag version"
             exit 1
           fi
           echo "✅ Version check passed: $TAG_VERSION"
-      
+
       - name: Install build dependencies
         run: |
           python -m pip install --upgrade pip
           pip install build twine
-      
+
       - name: Build package
         run: python -m build
-      
+
       - name: Check package
         run: twine check dist/*
-      
+
       - name: Upload to PyPI
         env:
           TWINE_USERNAME: __token__
@@ -65,37 +65,7 @@ jobs:
           echo "📦 Uploading to PyPI..."
           twine upload dist/*
           echo "✅ Package uploaded to https://pypi.org/project/crawl4ai/"
-      
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-      
-      - name: Extract major and minor versions
-        id: versions
-        run: |
-          VERSION=${{ steps.get_version.outputs.VERSION }}
-          MAJOR=$(echo $VERSION | cut -d. -f1)
-          MINOR=$(echo $VERSION | cut -d. -f1-2)
-          echo "MAJOR=$MAJOR" >> $GITHUB_OUTPUT
-          echo "MINOR=$MINOR" >> $GITHUB_OUTPUT
-      
-      - name: Build and push Docker images
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          tags: |
-            unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
-            unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}
-            unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}
-            unclecode/crawl4ai:latest
-          platforms: linux/amd64,linux/arm64
-      
+
       - name: Create GitHub Release
         uses: softprops/action-gh-release@v2
         with:
@@ -103,26 +73,29 @@ jobs:
           name: Release v${{ steps.get_version.outputs.VERSION }}
           body: |
             ## 🎉 Crawl4AI v${{ steps.get_version.outputs.VERSION }} Released!
-            
+
             ### 📦 Installation
-            
+
             **PyPI:**
             ```bash
             pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}
             ```
-            
+
             **Docker:**
             ```bash
             docker pull unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
             docker pull unclecode/crawl4ai:latest
             ```
-            
+
+            **Note:** Docker images are being built and will be available shortly.
+            Check the [Docker Release workflow](https://github.com/${{ github.repository }}/actions/workflows/docker-release.yml) for build status.
+
             ### 📝 What's Changed
             See [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/main/CHANGELOG.md) for details.
           draft: false
           prerelease: false
           token: ${{ secrets.GITHUB_TOKEN }}
-      
+
       - name: Summary
         run: |
           echo "## 🚀 Release Complete!" >> $GITHUB_STEP_SUMMARY
@@ -132,11 +105,9 @@ jobs:
           echo "- URL: https://pypi.org/project/crawl4ai/" >> $GITHUB_STEP_SUMMARY
           echo "- Install: \`pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### 🐳 Docker Images" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:latest\`" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
           echo "### 📋 GitHub Release" >> $GITHUB_STEP_SUMMARY
-          echo "https://github.com/${{ github.repository }}/releases/tag/v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "- https://github.com/${{ github.repository }}/releases/tag/v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 🐳 Docker Images" >> $GITHUB_STEP_SUMMARY
+          echo "Docker images are being built in a separate workflow." >> $GITHUB_STEP_SUMMARY
+          echo "Check: https://github.com/${{ github.repository }}/actions/workflows/docker-release.yml" >> $GITHUB_STEP_SUMMARY

.github/workflows/release.yml.backup

@@ -0,0 +1,142 @@
+name: Release Pipeline
+on:
+  push:
+    tags:
+      - 'v*'
+      - '!test-v*'  # Exclude test tags
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write  # Required for creating releases
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      
+      - name: Extract version from tag
+        id: get_version
+        run: |
+          TAG_VERSION=${GITHUB_REF#refs/tags/v}
+          echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT
+          echo "Releasing version: $TAG_VERSION"
+      
+      - name: Install package dependencies
+        run: |
+          pip install -e .
+      
+      - name: Check version consistency
+        run: |
+          TAG_VERSION=${{ steps.get_version.outputs.VERSION }}
+          PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)")
+          
+          echo "Tag version: $TAG_VERSION"
+          echo "Package version: $PACKAGE_VERSION"
+          
+          if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then
+            echo "❌ Version mismatch! Tag: $TAG_VERSION, Package: $PACKAGE_VERSION"
+            echo "Please update crawl4ai/__version__.py to match the tag version"
+            exit 1
+          fi
+          echo "✅ Version check passed: $TAG_VERSION"
+      
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build twine
+      
+      - name: Build package
+        run: python -m build
+      
+      - name: Check package
+        run: twine check dist/*
+      
+      - name: Upload to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          echo "📦 Uploading to PyPI..."
+          twine upload dist/*
+          echo "✅ Package uploaded to https://pypi.org/project/crawl4ai/"
+      
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      
+      - name: Extract major and minor versions
+        id: versions
+        run: |
+          VERSION=${{ steps.get_version.outputs.VERSION }}
+          MAJOR=$(echo $VERSION | cut -d. -f1)
+          MINOR=$(echo $VERSION | cut -d. -f1-2)
+          echo "MAJOR=$MAJOR" >> $GITHUB_OUTPUT
+          echo "MINOR=$MINOR" >> $GITHUB_OUTPUT
+      
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
+            unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}
+            unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}
+            unclecode/crawl4ai:latest
+          platforms: linux/amd64,linux/arm64
+      
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: v${{ steps.get_version.outputs.VERSION }}
+          name: Release v${{ steps.get_version.outputs.VERSION }}
+          body: |
+            ## 🎉 Crawl4AI v${{ steps.get_version.outputs.VERSION }} Released!
+            
+            ### 📦 Installation
+            
+            **PyPI:**
+            ```bash
+            pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}
+            ```
+            
+            **Docker:**
+            ```bash
+            docker pull unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
+            docker pull unclecode/crawl4ai:latest
+            ```
+            
+            ### 📝 What's Changed
+            See [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/main/CHANGELOG.md) for details.
+          draft: false
+          prerelease: false
+          token: ${{ secrets.GITHUB_TOKEN }}
+      
+      - name: Summary
+        run: |
+          echo "## 🚀 Release Complete!" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 📦 PyPI Package" >> $GITHUB_STEP_SUMMARY
+          echo "- Version: ${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "- URL: https://pypi.org/project/crawl4ai/" >> $GITHUB_STEP_SUMMARY
+          echo "- Install: \`pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 🐳 Docker Images" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:latest\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 📋 GitHub Release" >> $GITHUB_STEP_SUMMARY
+          echo "https://github.com/${{ github.repository }}/releases/tag/v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -1,8 +1,12 @@
 # Scripts folder (private tools)
 .scripts/
 
-# Docker automation scripts (personal use)
-docker-scripts/
+# Database files
+*.db
+
+# Environment files
+.env
+.env.local
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -262,6 +266,8 @@ continue_config.json
 .llm.env
 .private/
 
+.claude/
+
 CLAUDE_MONITOR.md
 CLAUDE.md
 
@@ -274,6 +280,5 @@ docs/**/data
 
 docs/apps/linkdin/debug*/
 docs/apps/linkdin/samples/insights/*
-.yoyo/
-.github/instructions/instructions.instructions.md
-.kilocode/mcp.json
+
+scripts/

Dockerfile

@@ -124,7 +124,7 @@ COPY . /tmp/project/
 
 # Copy supervisor config first (might need root later, but okay for now)
 COPY deploy/docker/supervisord.conf .
-COPY deploy/docker/routers ./routers
+
 COPY deploy/docker/requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 

crawl4ai/__init__.py

@@ -25,8 +25,7 @@ from .extraction_strategy import (
     JsonCssExtractionStrategy,
     JsonXPathExtractionStrategy,
     JsonLxmlExtractionStrategy,
-    RegexExtractionStrategy,
-    NoExtractionStrategy,  # NEW: Import NoExtractionStrategy
+    RegexExtractionStrategy
 )
 from .chunking_strategy import ChunkingStrategy, RegexChunking
 from .markdown_generation_strategy import DefaultMarkdownGenerator
@@ -104,7 +103,8 @@ from .browser_adapter import (
 
 from .utils import (
     start_colab_display_server,
-    setup_colab_environment
+    setup_colab_environment,
+    hooks_to_string
 )
 
 __all__ = [
@@ -114,7 +114,6 @@ __all__ = [
     "BrowserProfiler",
     "LLMConfig",
     "GeolocationConfig",
-    "NoExtractionStrategy",
     # NEW: Add SeedingConfig and VirtualScrollConfig
     "SeedingConfig",
     "VirtualScrollConfig",
@@ -185,6 +184,7 @@ __all__ = [
     "ProxyConfig",
     "start_colab_display_server",
     "setup_colab_environment",
+    "hooks_to_string",
     # C4A Script additions
     "c4a_compile",
     "c4a_validate", 

crawl4ai/async_dispatcher.py

@@ -455,8 +455,6 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
                     
                 # Update priorities for waiting tasks if needed
                 await self._update_queue_priorities()
-                
-            return results
 
         except Exception as e:
             if self.monitor:
@@ -467,6 +465,7 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
             memory_monitor.cancel()
             if self.monitor:
                 self.monitor.stop()
+            return results
                 
     async def _update_queue_priorities(self):
         """Periodically update priorities of items in the queue to prevent starvation"""

crawl4ai/docker_client.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Union, AsyncGenerator, Dict, Any
+from typing import List, Optional, Union, AsyncGenerator, Dict, Any, Callable
 import httpx
 import json
 from urllib.parse import urljoin
@@ -7,6 +7,7 @@ import asyncio
 from .async_configs import BrowserConfig, CrawlerRunConfig
 from .models import CrawlResult
 from .async_logger import AsyncLogger, LogLevel
+from .utils import hooks_to_string
 
 
 class Crawl4aiClientError(Exception):
@@ -70,17 +71,41 @@ class Crawl4aiDockerClient:
             self.logger.error(f"Server unreachable: {str(e)}", tag="ERROR")
             raise ConnectionError(f"Cannot connect to server: {str(e)}")
 
-    def _prepare_request(self, urls: List[str], browser_config: Optional[BrowserConfig] = None, 
-                       crawler_config: Optional[CrawlerRunConfig] = None) -> Dict[str, Any]:
+    def _prepare_request(
+        self,
+        urls: List[str],
+        browser_config: Optional[BrowserConfig] = None,
+        crawler_config: Optional[CrawlerRunConfig] = None,
+        hooks: Optional[Union[Dict[str, Callable], Dict[str, str]]] = None,
+        hooks_timeout: int = 30
+    ) -> Dict[str, Any]:
         """Prepare request data from configs."""
         if self._token:
             self._http_client.headers["Authorization"] = f"Bearer {self._token}"
-        return {
+
+        request_data = {
             "urls": urls,
             "browser_config": browser_config.dump() if browser_config else {},
             "crawler_config": crawler_config.dump() if crawler_config else {}
         }
 
+        # Handle hooks if provided
+        if hooks:
+            # Check if hooks are already strings or need conversion
+            if any(callable(v) for v in hooks.values()):
+                # Convert function objects to strings
+                hooks_code = hooks_to_string(hooks)
+            else:
+                # Already in string format
+                hooks_code = hooks
+
+            request_data["hooks"] = {
+                "code": hooks_code,
+                "timeout": hooks_timeout
+            }
+
+        return request_data
+
     async def _request(self, method: str, endpoint: str, **kwargs) -> httpx.Response:
         """Make an HTTP request with error handling."""
         url = urljoin(self.base_url, endpoint)
@@ -102,16 +127,42 @@ class Crawl4aiDockerClient:
         self,
         urls: List[str],
         browser_config: Optional[BrowserConfig] = None,
-        crawler_config: Optional[CrawlerRunConfig] = None
+        crawler_config: Optional[CrawlerRunConfig] = None,
+        hooks: Optional[Union[Dict[str, Callable], Dict[str, str]]] = None,
+        hooks_timeout: int = 30
     ) -> Union[CrawlResult, List[CrawlResult], AsyncGenerator[CrawlResult, None]]:
-        """Execute a crawl operation."""
+        """
+        Execute a crawl operation.
+
+        Args:
+            urls: List of URLs to crawl
+            browser_config: Browser configuration
+            crawler_config: Crawler configuration
+            hooks: Optional hooks - can be either:
+                   - Dict[str, Callable]: Function objects that will be converted to strings
+                   - Dict[str, str]: Already stringified hook code
+            hooks_timeout: Timeout in seconds for each hook execution (1-120)
+
+        Returns:
+            Single CrawlResult, list of results, or async generator for streaming
+
+        Example with function hooks:
+            >>> async def my_hook(page, context, **kwargs):
+            ...     await page.set_viewport_size({"width": 1920, "height": 1080})
+            ...     return page
+            >>>
+            >>> result = await client.crawl(
+            ...     ["https://example.com"],
+            ...     hooks={"on_page_context_created": my_hook}
+            ... )
+        """
         await self._check_server()
-        
-        data = self._prepare_request(urls, browser_config, crawler_config)
+
+        data = self._prepare_request(urls, browser_config, crawler_config, hooks, hooks_timeout)
         is_streaming = crawler_config and crawler_config.stream
-        
+
         self.logger.info(f"Crawling {len(urls)} URLs {'(streaming)' if is_streaming else ''}", tag="CRAWL")
-        
+
         if is_streaming:
             async def stream_results() -> AsyncGenerator[CrawlResult, None]:
                 async with self._http_client.stream("POST", f"{self.base_url}/crawl/stream", json=data) as response:
@@ -128,12 +179,12 @@ class Crawl4aiDockerClient:
                             else:
                                 yield CrawlResult(**result)
             return stream_results()
-        
+
         response = await self._request("POST", "/crawl", json=data)
         result_data = response.json()
         if not result_data.get("success", False):
             raise RequestError(f"Crawl failed: {result_data.get('msg', 'Unknown error')}")
-        
+
         results = [CrawlResult(**r) for r in result_data.get("results", [])]
         self.logger.success(f"Crawl completed with {len(results)} results", tag="CRAWL")
         return results[0] if len(results) == 1 else results

crawl4ai/proxy_strategy.py

@@ -2,11 +2,6 @@ from typing import List, Dict, Optional
 from abc import ABC, abstractmethod
 from itertools import cycle
 import os
-import random
-import time
-import asyncio
-import logging
-from collections import defaultdict
 
 
 ########### ATTENTION PEOPLE OF EARTH ###########
@@ -136,7 +131,7 @@ class ProxyRotationStrategy(ABC):
         """Add proxy configurations to the strategy"""
         pass
 
-class RoundRobinProxyStrategy(ProxyRotationStrategy):
+class RoundRobinProxyStrategy:
     """Simple round-robin proxy rotation strategy using ProxyConfig objects"""
 
     def __init__(self, proxies: List[ProxyConfig] = None):
@@ -161,113 +156,3 @@ class RoundRobinProxyStrategy(ProxyRotationStrategy):
         if not self._proxy_cycle:
             return None
         return next(self._proxy_cycle)
-
-
-class RandomProxyStrategy(ProxyRotationStrategy):
-    """Random proxy selection strategy for unpredictable traffic patterns."""
-    
-    def __init__(self, proxies: List[ProxyConfig] = None):
-        self._proxies = []
-        self._lock = asyncio.Lock()
-        if proxies:
-            self.add_proxies(proxies)
-    
-    def add_proxies(self, proxies: List[ProxyConfig]):
-        """Add new proxies to the rotation pool."""
-        self._proxies.extend(proxies)
-    
-    async def get_next_proxy(self) -> Optional[ProxyConfig]:
-        """Get randomly selected proxy."""
-        async with self._lock:
-            if not self._proxies:
-                return None
-            return random.choice(self._proxies)
-
-
-class LeastUsedProxyStrategy(ProxyRotationStrategy):
-    """Least used proxy strategy for optimal load distribution."""
-    
-    def __init__(self, proxies: List[ProxyConfig] = None):
-        self._proxies = []
-        self._usage_count: Dict[str, int] = defaultdict(int)
-        self._lock = asyncio.Lock()
-        if proxies:
-            self.add_proxies(proxies)
-    
-    def add_proxies(self, proxies: List[ProxyConfig]):
-        """Add new proxies to the rotation pool."""
-        self._proxies.extend(proxies)
-        for proxy in proxies:
-            self._usage_count[proxy.server] = 0
-    
-    async def get_next_proxy(self) -> Optional[ProxyConfig]:
-        """Get least used proxy for optimal load balancing."""
-        async with self._lock:
-            if not self._proxies:
-                return None
-            
-            # Find proxy with minimum usage
-            min_proxy = min(self._proxies, key=lambda p: self._usage_count[p.server])
-            self._usage_count[min_proxy.server] += 1
-            return min_proxy
-
-
-class FailureAwareProxyStrategy(ProxyRotationStrategy):
-    """Failure-aware proxy strategy with automatic recovery and health tracking."""
-    
-    def __init__(self, proxies: List[ProxyConfig] = None, failure_threshold: int = 3, recovery_time: int = 300):
-        self._proxies = []
-        self._healthy_proxies = []
-        self._failure_count: Dict[str, int] = defaultdict(int)
-        self._last_failure_time: Dict[str, float] = defaultdict(float)
-        self._failure_threshold = failure_threshold
-        self._recovery_time = recovery_time  # seconds
-        self._lock = asyncio.Lock()
-        if proxies:
-            self.add_proxies(proxies)
-    
-    def add_proxies(self, proxies: List[ProxyConfig]):
-        """Add new proxies to the rotation pool."""
-        self._proxies.extend(proxies)
-        self._healthy_proxies.extend(proxies)
-        for proxy in proxies:
-            self._failure_count[proxy.server] = 0
-    
-    async def get_next_proxy(self) -> Optional[ProxyConfig]:
-        """Get next healthy proxy with automatic recovery."""
-        async with self._lock:
-            # Recovery check: re-enable proxies after recovery_time
-            current_time = time.time()
-            recovered_proxies = []
-            
-            for proxy in self._proxies:
-                if (proxy not in self._healthy_proxies and 
-                    current_time - self._last_failure_time[proxy.server] > self._recovery_time):
-                    recovered_proxies.append(proxy)
-                    self._failure_count[proxy.server] = 0
-            
-            # Add recovered proxies back to healthy pool
-            self._healthy_proxies.extend(recovered_proxies)
-            
-            # If no healthy proxies, reset all (emergency fallback)
-            if not self._healthy_proxies and self._proxies:
-                logging.warning("All proxies failed, resetting health status")
-                self._healthy_proxies = self._proxies.copy()
-                for proxy in self._proxies:
-                    self._failure_count[proxy.server] = 0
-            
-            if not self._healthy_proxies:
-                return None
-                
-            return random.choice(self._healthy_proxies)
-    
-    async def mark_proxy_failed(self, proxy: ProxyConfig):
-        """Mark a proxy as failed and remove from healthy pool if threshold exceeded."""
-        async with self._lock:
-            self._failure_count[proxy.server] += 1
-            self._last_failure_time[proxy.server] = time.time()
-            
-            if (self._failure_count[proxy.server] >= self._failure_threshold and 
-                proxy in self._healthy_proxies):
-                self._healthy_proxies.remove(proxy)
-                logging.warning(f"Proxy {proxy.server} marked as unhealthy after {self._failure_count[proxy.server]} failures")

crawl4ai/types_backup.py

@@ -1,195 +0,0 @@
-from typing import TYPE_CHECKING, Union
-
-# Logger types
-AsyncLoggerBase = Union['AsyncLoggerBaseType']
-AsyncLogger = Union['AsyncLoggerType']
-
-# Crawler core types
-AsyncWebCrawler = Union['AsyncWebCrawlerType']
-CacheMode = Union['CacheModeType']
-CrawlResult = Union['CrawlResultType']
-CrawlerHub = Union['CrawlerHubType']
-BrowserProfiler = Union['BrowserProfilerType']
-# NEW: Add AsyncUrlSeederType
-AsyncUrlSeeder = Union['AsyncUrlSeederType']
-
-# Configuration types
-BrowserConfig = Union['BrowserConfigType']
-CrawlerRunConfig = Union['CrawlerRunConfigType']
-HTTPCrawlerConfig = Union['HTTPCrawlerConfigType']
-LLMConfig = Union['LLMConfigType']
-# NEW: Add SeedingConfigType
-SeedingConfig = Union['SeedingConfigType']
-
-# Content scraping types
-ContentScrapingStrategy = Union['ContentScrapingStrategyType']
-LXMLWebScrapingStrategy = Union['LXMLWebScrapingStrategyType']
-# Backward compatibility alias
-WebScrapingStrategy = Union['LXMLWebScrapingStrategyType']
-
-# Proxy types
-ProxyRotationStrategy = Union['ProxyRotationStrategyType']
-RoundRobinProxyStrategy = Union['RoundRobinProxyStrategyType']
-
-# Extraction types
-ExtractionStrategy = Union['ExtractionStrategyType']
-LLMExtractionStrategy = Union['LLMExtractionStrategyType']
-CosineStrategy = Union['CosineStrategyType']
-JsonCssExtractionStrategy = Union['JsonCssExtractionStrategyType']
-JsonXPathExtractionStrategy = Union['JsonXPathExtractionStrategyType']
-
-# Chunking types
-ChunkingStrategy = Union['ChunkingStrategyType']
-RegexChunking = Union['RegexChunkingType']
-
-# Markdown generation types
-DefaultMarkdownGenerator = Union['DefaultMarkdownGeneratorType']
-MarkdownGenerationResult = Union['MarkdownGenerationResultType']
-
-# Content filter types
-RelevantContentFilter = Union['RelevantContentFilterType']
-PruningContentFilter = Union['PruningContentFilterType']
-BM25ContentFilter = Union['BM25ContentFilterType']
-LLMContentFilter = Union['LLMContentFilterType']
-
-# Dispatcher types
-BaseDispatcher = Union['BaseDispatcherType']
-MemoryAdaptiveDispatcher = Union['MemoryAdaptiveDispatcherType']
-SemaphoreDispatcher = Union['SemaphoreDispatcherType']
-RateLimiter = Union['RateLimiterType']
-CrawlerMonitor = Union['CrawlerMonitorType']
-DisplayMode = Union['DisplayModeType']
-RunManyReturn = Union['RunManyReturnType']
-
-# Docker client
-Crawl4aiDockerClient = Union['Crawl4aiDockerClientType']
-
-# Deep crawling types
-DeepCrawlStrategy = Union['DeepCrawlStrategyType']
-BFSDeepCrawlStrategy = Union['BFSDeepCrawlStrategyType']
-FilterChain = Union['FilterChainType']
-ContentTypeFilter = Union['ContentTypeFilterType']
-DomainFilter = Union['DomainFilterType']
-URLFilter = Union['URLFilterType']
-FilterStats = Union['FilterStatsType']
-SEOFilter = Union['SEOFilterType']
-KeywordRelevanceScorer = Union['KeywordRelevanceScorerType']
-URLScorer = Union['URLScorerType']
-CompositeScorer = Union['CompositeScorerType']
-DomainAuthorityScorer = Union['DomainAuthorityScorerType']
-FreshnessScorer = Union['FreshnessScorerType']
-PathDepthScorer = Union['PathDepthScorerType']
-BestFirstCrawlingStrategy = Union['BestFirstCrawlingStrategyType']
-DFSDeepCrawlStrategy = Union['DFSDeepCrawlStrategyType']
-DeepCrawlDecorator = Union['DeepCrawlDecoratorType']
-
-# Only import types during type checking to avoid circular imports
-if TYPE_CHECKING:
-    # Logger imports
-    from .async_logger import (
-        AsyncLoggerBase as AsyncLoggerBaseType,
-        AsyncLogger as AsyncLoggerType,
-    )
-    
-    # Crawler core imports
-    from .async_webcrawler import (
-        AsyncWebCrawler as AsyncWebCrawlerType,
-        CacheMode as CacheModeType,
-    )
-    from .models import CrawlResult as CrawlResultType
-    from .hub import CrawlerHub as CrawlerHubType
-    from .browser_profiler import BrowserProfiler as BrowserProfilerType
-    # NEW: Import AsyncUrlSeeder for type checking
-    from .async_url_seeder import AsyncUrlSeeder as AsyncUrlSeederType
-    
-    # Configuration imports
-    from .async_configs import (
-        BrowserConfig as BrowserConfigType,
-        CrawlerRunConfig as CrawlerRunConfigType,
-        HTTPCrawlerConfig as HTTPCrawlerConfigType,
-        LLMConfig as LLMConfigType,
-        # NEW: Import SeedingConfig for type checking
-        SeedingConfig as SeedingConfigType,
-    )
-    
-    # Content scraping imports
-    from .content_scraping_strategy import (
-        ContentScrapingStrategy as ContentScrapingStrategyType,
-        LXMLWebScrapingStrategy as LXMLWebScrapingStrategyType,
-    )
-    
-    # Proxy imports
-    from .proxy_strategy import (
-        ProxyRotationStrategy as ProxyRotationStrategyType,
-        RoundRobinProxyStrategy as RoundRobinProxyStrategyType,
-    )
-    
-    # Extraction imports
-    from .extraction_strategy import (
-        ExtractionStrategy as ExtractionStrategyType,
-        LLMExtractionStrategy as LLMExtractionStrategyType,
-        CosineStrategy as CosineStrategyType,
-        JsonCssExtractionStrategy as JsonCssExtractionStrategyType,
-        JsonXPathExtractionStrategy as JsonXPathExtractionStrategyType,
-    )
-    
-    # Chunking imports
-    from .chunking_strategy import (
-        ChunkingStrategy as ChunkingStrategyType,
-        RegexChunking as RegexChunkingType,
-    )
-    
-    # Markdown generation imports
-    from .markdown_generation_strategy import (
-        DefaultMarkdownGenerator as DefaultMarkdownGeneratorType,
-    )
-    from .models import MarkdownGenerationResult as MarkdownGenerationResultType
-    
-    # Content filter imports
-    from .content_filter_strategy import (
-        RelevantContentFilter as RelevantContentFilterType,
-        PruningContentFilter as PruningContentFilterType,
-        BM25ContentFilter as BM25ContentFilterType,
-        LLMContentFilter as LLMContentFilterType,
-    )
-    
-    # Dispatcher imports
-    from .async_dispatcher import (
-        BaseDispatcher as BaseDispatcherType,
-        MemoryAdaptiveDispatcher as MemoryAdaptiveDispatcherType,
-        SemaphoreDispatcher as SemaphoreDispatcherType,
-        RateLimiter as RateLimiterType,
-        CrawlerMonitor as CrawlerMonitorType,
-        DisplayMode as DisplayModeType,
-        RunManyReturn as RunManyReturnType,
-    )
-    
-    # Docker client
-    from .docker_client import Crawl4aiDockerClient as Crawl4aiDockerClientType
-    
-    # Deep crawling imports
-    from .deep_crawling import (
-        DeepCrawlStrategy as DeepCrawlStrategyType,
-        BFSDeepCrawlStrategy as BFSDeepCrawlStrategyType,
-        FilterChain as FilterChainType,
-        ContentTypeFilter as ContentTypeFilterType,
-        DomainFilter as DomainFilterType,
-        URLFilter as URLFilterType,
-        FilterStats as FilterStatsType,
-        SEOFilter as SEOFilterType,
-        KeywordRelevanceScorer as KeywordRelevanceScorerType,
-        URLScorer as URLScorerType,
-        CompositeScorer as CompositeScorerType,
-        DomainAuthorityScorer as DomainAuthorityScorerType,
-        FreshnessScorer as FreshnessScorerType,
-        PathDepthScorer as PathDepthScorerType,
-        BestFirstCrawlingStrategy as BestFirstCrawlingStrategyType,
-        DFSDeepCrawlStrategy as DFSDeepCrawlStrategyType,
-        DeepCrawlDecorator as DeepCrawlDecoratorType,
-    )
-
-
-
-def create_llm_config(*args, **kwargs) -> 'LLMConfigType':
-    from .async_configs import LLMConfig
-    return LLMConfig(*args, **kwargs)

crawl4ai/utils.py

@@ -47,6 +47,7 @@ from urllib.parse import (
     urljoin, urlparse, urlunparse,
     parse_qsl, urlencode, quote, unquote
 )
+import inspect
 
 
 # Monkey patch to fix wildcard handling in urllib.robotparser
@@ -3529,4 +3530,52 @@ def get_memory_stats() -> Tuple[float, float, float]:
     available_gb = get_true_available_memory_gb()
     used_percent = get_true_memory_usage_percent()
     
-    return used_percent, available_gb, total_gb
+    return used_percent, available_gb, total_gb
+
+
+# Hook utilities for Docker API
+def hooks_to_string(hooks: Dict[str, Callable]) -> Dict[str, str]:
+    """
+    Convert hook function objects to string representations for Docker API.
+
+    This utility simplifies the process of using hooks with the Docker API by converting
+    Python function objects into the string format required by the API.
+
+    Args:
+        hooks: Dictionary mapping hook point names to Python function objects.
+               Functions should be async and follow hook signature requirements.
+
+    Returns:
+        Dictionary mapping hook point names to string representations of the functions.
+
+    Example:
+        >>> async def my_hook(page, context, **kwargs):
+        ...     await page.set_viewport_size({"width": 1920, "height": 1080})
+        ...     return page
+        >>>
+        >>> hooks_dict = {"on_page_context_created": my_hook}
+        >>> api_hooks = hooks_to_string(hooks_dict)
+        >>> # api_hooks is now ready to use with Docker API
+
+    Raises:
+        ValueError: If a hook is not callable or source cannot be extracted
+    """
+    result = {}
+
+    for hook_name, hook_func in hooks.items():
+        if not callable(hook_func):
+            raise ValueError(f"Hook '{hook_name}' must be a callable function, got {type(hook_func)}")
+
+        try:
+            # Get the source code of the function
+            source = inspect.getsource(hook_func)
+            # Remove any leading indentation to get clean source
+            source = textwrap.dedent(source)
+            result[hook_name] = source
+        except (OSError, TypeError) as e:
+            raise ValueError(
+                f"Cannot extract source code for hook '{hook_name}'. "
+                f"Make sure the function is defined in a file (not interactively). Error: {e}"
+            )
+
+    return result

deploy/docker/README.md

@@ -13,7 +13,6 @@
   - [Understanding Request Schema](#understanding-request-schema)
   - [REST API Examples](#rest-api-examples)
 - [Additional API Endpoints](#additional-api-endpoints)
-  - [Dispatcher Management](#dispatcher-management)
   - [HTML Extraction Endpoint](#html-extraction-endpoint)
   - [Screenshot Endpoint](#screenshot-endpoint)
   - [PDF Export Endpoint](#pdf-export-endpoint)
@@ -35,8 +34,6 @@
   - [Configuration Tips and Best Practices](#configuration-tips-and-best-practices)
   - [Customizing Your Configuration](#customizing-your-configuration)
   - [Configuration Recommendations](#configuration-recommendations)
-- [Testing & Validation](#testing--validation)
-  - [Dispatcher Demo Test Suite](#dispatcher-demo-test-suite)
 - [Getting Help](#getting-help)
 - [Summary](#summary)
 
@@ -335,134 +332,6 @@ Access the MCP tool schemas at `http://localhost:11235/mcp/schema` for detailed
 
 In addition to the core `/crawl` and `/crawl/stream` endpoints, the server provides several specialized endpoints:
 
-### Dispatcher Management
-
-The server supports multiple dispatcher strategies for managing concurrent crawling operations. Dispatchers control how many crawl jobs run simultaneously based on different rules like fixed concurrency limits or system memory availability.
-
-#### Available Dispatchers
-
-**Memory Adaptive Dispatcher** (Default)
-- Dynamically adjusts concurrency based on system memory usage
-- Monitors memory pressure and adapts crawl sessions accordingly
-- Automatically requeues tasks under high memory conditions
-- Implements fairness timeout for long-waiting URLs
-
-**Semaphore Dispatcher**
-- Fixed concurrency limit using semaphore-based control
-- Simple and predictable resource usage
-- Ideal for controlled crawling scenarios
-
-#### Dispatcher Endpoints
-
-**List Available Dispatchers**
-```bash
-GET /dispatchers
-```
-
-Returns information about all available dispatcher types, their configurations, and features.
-
-```bash
-curl http://localhost:11234/dispatchers | jq
-```
-
-**Get Default Dispatcher**
-```bash
-GET /dispatchers/default
-```
-
-Returns the current default dispatcher configuration.
-
-```bash
-curl http://localhost:11234/dispatchers/default | jq
-```
-
-**Get Dispatcher Statistics**
-```bash
-GET /dispatchers/{dispatcher_type}/stats
-```
-
-Returns real-time statistics for a specific dispatcher including active sessions, memory usage, and configuration.
-
-```bash
-# Get memory_adaptive dispatcher stats
-curl http://localhost:11234/dispatchers/memory_adaptive/stats | jq
-
-# Get semaphore dispatcher stats
-curl http://localhost:11234/dispatchers/semaphore/stats | jq
-```
-
-#### Using Dispatchers in Crawl Requests
-
-You can specify which dispatcher to use in your crawl requests by adding the `dispatcher` field:
-
-**Using Default Dispatcher (memory_adaptive)**
-```bash
-curl -X POST http://localhost:11234/crawl \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://example.com"],
-    "browser_config": {},
-    "crawler_config": {}
-  }'
-```
-
-**Using Semaphore Dispatcher**
-```bash
-curl -X POST http://localhost:11234/crawl \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://example.com", "https://httpbin.org/html"],
-    "browser_config": {},
-    "crawler_config": {},
-    "dispatcher": "semaphore"
-  }'
-```
-
-**Python SDK Example**
-```python
-import requests
-
-# Crawl with memory adaptive dispatcher (default)
-response = requests.post(
-    "http://localhost:11234/crawl",
-    json={
-        "urls": ["https://example.com"],
-        "browser_config": {},
-        "crawler_config": {}
-    }
-)
-
-# Crawl with semaphore dispatcher
-response = requests.post(
-    "http://localhost:11234/crawl",
-    json={
-        "urls": ["https://example.com"],
-        "browser_config": {},
-        "crawler_config": {},
-        "dispatcher": "semaphore"
-    }
-)
-```
-
-#### Dispatcher Configuration
-
-Dispatchers are configured with sensible defaults that work well for most use cases:
-
-**Memory Adaptive Dispatcher Defaults:**
-- `memory_threshold_percent`: 70.0 - Start adjusting at 70% memory usage
-- `critical_threshold_percent`: 85.0 - Critical memory pressure threshold
-- `recovery_threshold_percent`: 65.0 - Resume normal operation below 65%
-- `check_interval`: 1.0 - Check memory every second
-- `max_session_permit`: 20 - Maximum concurrent sessions
-- `fairness_timeout`: 600.0 - Prioritize URLs waiting > 10 minutes
-- `memory_wait_timeout`: 600.0 - Fail if high memory persists > 10 minutes
-
-**Semaphore Dispatcher Defaults:**
-- `semaphore_count`: 5 - Maximum concurrent crawl operations
-- `max_session_permit`: 10 - Maximum total sessions allowed
-
-> 💡 **Tip**: Use `memory_adaptive` for dynamic workloads where memory availability varies. Use `semaphore` for predictable, controlled crawling with fixed concurrency limits.
-
 ### HTML Extraction Endpoint
 
 ```
@@ -779,144 +648,6 @@ async def test_stream_crawl(token: str = None): # Made token optional
 # asyncio.run(test_stream_crawl())
 ```
 
-#### LLM Job with Chunking Strategy
-
-```python
-import requests
-import time
-
-# Example: LLM extraction with RegexChunking strategy
-# This breaks large documents into smaller chunks before LLM processing
-
-llm_job_payload = {
-    "url": "https://example.com/long-article",
-    "q": "Extract all key points and main ideas from this article",
-    "chunking_strategy": {
-        "type": "RegexChunking",
-        "params": {
-            "patterns": ["\\n\\n"],  # Split on double newlines (paragraphs)
-            "overlap": 50
-        }
-    }
-}
-
-# Submit LLM job
-response = requests.post(
-    "http://localhost:11235/llm/job",
-    json=llm_job_payload
-)
-
-if response.ok:
-    job_data = response.json()
-    job_id = job_data["task_id"]
-    print(f"Job submitted successfully. Job ID: {job_id}")
-    
-    # Poll for completion
-    while True:
-        status_response = requests.get(f"http://localhost:11235/llm/job/{job_id}")
-        if status_response.ok:
-            status_data = status_response.json()
-            if status_data["status"] == "completed":
-                print("Job completed!")
-                print("Extracted content:", status_data["result"])
-                break
-            elif status_data["status"] == "failed":
-                print("Job failed:", status_data.get("error"))
-                break
-            else:
-                print(f"Job status: {status_data['status']}")
-                time.sleep(2)  # Wait 2 seconds before checking again
-        else:
-            print(f"Error checking job status: {status_response.text}")
-            break
-else:
-    print(f"Error submitting job: {response.text}")
-```
-
-**Available Chunking Strategies:**
-
-- **IdentityChunking**: Returns the entire content as a single chunk (no splitting)
-  ```json
-  {
-    "type": "IdentityChunking",
-    "params": {}
-  }
-  ```
-
-- **RegexChunking**: Split content using regular expression patterns
-  ```json
-  {
-    "type": "RegexChunking",
-    "params": {
-      "patterns": ["\\n\\n"]
-    }
-  }
-  ```
-
-- **NlpSentenceChunking**: Split content into sentences using NLP (requires NLTK)
-  ```json
-  {
-    "type": "NlpSentenceChunking",
-    "params": {}
-  }
-  ```
-
-- **TopicSegmentationChunking**: Segment content into topics using TextTiling (requires NLTK)
-  ```json
-  {
-    "type": "TopicSegmentationChunking",
-    "params": {
-      "num_keywords": 3
-    }
-  }
-  ```
-
-- **FixedLengthWordChunking**: Split into fixed-length word chunks
-  ```json
-  {
-    "type": "FixedLengthWordChunking",
-    "params": {
-      "chunk_size": 100
-    }
-  }
-  ```
-
-- **SlidingWindowChunking**: Overlapping word chunks with configurable step size
-  ```json
-  {
-    "type": "SlidingWindowChunking",
-    "params": {
-      "window_size": 100,
-      "step": 50
-    }
-  }
-  ```
-
-- **OverlappingWindowChunking**: Fixed-size chunks with word overlap
-  ```json
-  {
-    "type": "OverlappingWindowChunking",
-    "params": {
-      "window_size": 1000,
-      "overlap": 100
-    }
-  }
-  ```
-  {
-    "type": "OverlappingWindowChunking", 
-    "params": {
-      "chunk_size": 1500,
-      "overlap": 100
-    }
-  }
-  ```
-
-**Notes:**
-- `chunking_strategy` is optional - if omitted, default token-based chunking is used
-- Chunking is applied at the API level without modifying the core SDK
-- Results from all chunks are merged into a single response
-- Each chunk is processed independently with the same LLM instruction
-
 ---
 
 ## Metrics & Monitoring
@@ -1082,93 +813,6 @@ You can override the default `config.yml`.
    - Increase batch_process timeout for large content
    - Adjust stream_init timeout based on initial response times
 
-## Testing & Validation
-
-We provide two comprehensive test suites to validate all Docker server functionality:
-
-### 1. Extended Features Test Suite ✅ **100% Pass Rate**
-
-Complete validation of all advanced features including URL seeding, adaptive crawling, browser adapters, proxy rotation, and dispatchers.
-
-```bash
-# Run all extended features tests
-cd tests/docker/extended_features
-./run_extended_tests.sh
-
-# Custom server URL
-./run_extended_tests.sh --server http://localhost:8080
-```
-
-**Test Coverage (12 tests):**
-- ✅ **URL Seeding** (2 tests): Basic seeding + domain filters
-- ✅ **Adaptive Crawling** (2 tests): Basic + custom thresholds  
-- ✅ **Browser Adapters** (3 tests): Default, Stealth, Undetected
-- ✅ **Proxy Rotation** (2 tests): Round Robin, Random strategies
-- ✅ **Dispatchers** (3 tests): Memory Adaptive, Semaphore, Management APIs
-
-**Current Status:**
-```
-Total Tests: 12
-Passed: 12
-Failed: 0
-Pass Rate: 100.0% ✅
-Average Duration: ~8.8 seconds
-```
-
-Features:
-- Rich formatted output with tables and panels
-- Real-time progress indicators
-- Detailed error diagnostics
-- Category-based results grouping
-- Server health checks
-
-See [`tests/docker/extended_features/README_EXTENDED_TESTS.md`](../../tests/docker/extended_features/README_EXTENDED_TESTS.md) for full documentation and API response format reference.
-
-### 2. Dispatcher Demo Test Suite
-
-Focused tests for dispatcher functionality with performance comparisons:
-
-```bash
-# Run all tests
-cd test_scripts
-./run_dispatcher_tests.sh
-
-# Run specific category
-./run_dispatcher_tests.sh -c basic          # Basic dispatcher usage
-./run_dispatcher_tests.sh -c integration   # Integration with other features
-./run_dispatcher_tests.sh -c endpoints     # Dispatcher management endpoints
-./run_dispatcher_tests.sh -c performance   # Performance comparison
-./run_dispatcher_tests.sh -c error         # Error handling
-
-# Custom server URL
-./run_dispatcher_tests.sh -s http://your-server:port
-```
-
-**Test Coverage (17 tests):**
-- **Basic Usage Tests**: Single/multiple URL crawling with different dispatchers
-- **Integration Tests**: Dispatchers combined with anti-bot strategies, browser configs, JS execution, screenshots
-- **Endpoint Tests**: Dispatcher management API validation
-- **Performance Tests**: Side-by-side comparison of memory_adaptive vs semaphore
-- **Error Handling**: Edge cases and validation tests
-
-Results are displayed with rich formatting, timing information, and success rates. See `test_scripts/README_DISPATCHER_TESTS.md` for full documentation.
-
-### Quick Test Commands
-
-```bash
-# Test all features (recommended)
-./tests/docker/extended_features/run_extended_tests.sh
-
-# Test dispatchers only
-./test_scripts/run_dispatcher_tests.sh
-
-# Test server health
-curl http://localhost:11235/health
-
-# Test dispatcher endpoint
-curl http://localhost:11235/dispatchers | jq
-```
-
 ## Getting Help
 
 We're here to help you succeed with Crawl4AI! Here's how to get support:

deploy/docker/crawler_pool.py

@@ -1,27 +1,10 @@
 # crawler_pool.py  (new file)
-import asyncio
-import hashlib
-import json
-import time
+import asyncio, json, hashlib, time, psutil
 from contextlib import suppress
-from typing import Dict, Optional
-
-import psutil
-
+from typing import Dict
 from crawl4ai import AsyncWebCrawler, BrowserConfig
-from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
-
-# Import browser adapters with fallback
-try:
-    from crawl4ai.browser_adapter import BrowserAdapter, PlaywrightAdapter
-except ImportError:
-    # Fallback for development environment
-    import os
-    import sys
-
-    sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
-    from crawl4ai.browser_adapter import BrowserAdapter, PlaywrightAdapter
-from utils import load_config
+from typing import Dict
+from utils import load_config 
 
 CONFIG = load_config()
 
@@ -29,82 +12,42 @@ POOL: Dict[str, AsyncWebCrawler] = {}
 LAST_USED: Dict[str, float] = {}
 LOCK = asyncio.Lock()
 
-MEM_LIMIT = CONFIG.get("crawler", {}).get(
-    "memory_threshold_percent", 95.0
-)  # % RAM – refuse new browsers above this
-IDLE_TTL = (
-    CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 1800)
-)  # close if unused for 30 min
+MEM_LIMIT  = CONFIG.get("crawler", {}).get("memory_threshold_percent", 95.0)   # % RAM – refuse new browsers above this
+IDLE_TTL  = CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 1800)   # close if unused for 30 min
 
-
-def _sig(cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None) -> str:
-    try:
-        config_payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",", ":"))
-    except (TypeError, ValueError):
-        # Fallback to string representation if JSON serialization fails
-        config_payload = str(cfg.to_dict())
-    adapter_name = adapter.__class__.__name__ if adapter else "PlaywrightAdapter"
-    payload = f"{config_payload}:{adapter_name}"
+def _sig(cfg: BrowserConfig) -> str:
+    payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",",":"))
     return hashlib.sha1(payload.encode()).hexdigest()
 
-
-async def get_crawler(
-    cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None
-) -> AsyncWebCrawler:
-    sig = None
+async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler:
     try:
-        sig = _sig(cfg, adapter)
+        sig = _sig(cfg)
         async with LOCK:
             if sig in POOL:
-                LAST_USED[sig] = time.time()
+                LAST_USED[sig] = time.time();  
                 return POOL[sig]
             if psutil.virtual_memory().percent >= MEM_LIMIT:
                 raise MemoryError("RAM pressure – new browser denied")
-
-            # Create crawler - let it initialize the strategy with proper logger
-            # Pass browser_adapter as a kwarg so AsyncWebCrawler can use it when creating the strategy
-            crawler = AsyncWebCrawler(
-                config=cfg, 
-                thread_safe=False
-            )
-            
-            # Set the browser adapter on the strategy after crawler initialization
-            if adapter:
-                # Create a new strategy with the adapter and the crawler's logger
-                from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
-                crawler.crawler_strategy = AsyncPlaywrightCrawlerStrategy(
-                    browser_config=cfg, 
-                    logger=crawler.logger,
-                    browser_adapter=adapter
-                )
-            
+            crawler = AsyncWebCrawler(config=cfg, thread_safe=False)
             await crawler.start()
-            POOL[sig] = crawler
-            LAST_USED[sig] = time.time()
+            POOL[sig] = crawler; LAST_USED[sig] = time.time()
             return crawler
     except MemoryError as e:
         raise MemoryError(f"RAM pressure – new browser denied: {e}")
     except Exception as e:
         raise RuntimeError(f"Failed to start browser: {e}")
     finally:
-        if sig:
-            if sig in POOL:
-                LAST_USED[sig] = time.time()
-            else:
-                # If we failed to start the browser, we should remove it from the pool
-                POOL.pop(sig, None)
-                LAST_USED.pop(sig, None)
+        if sig in POOL:
+            LAST_USED[sig] = time.time()
+        else:
+            # If we failed to start the browser, we should remove it from the pool
+            POOL.pop(sig, None)
+            LAST_USED.pop(sig, None)
         # If we failed to start the browser, we should remove it from the pool
-
-
 async def close_all():
     async with LOCK:
-        await asyncio.gather(
-            *(c.close() for c in POOL.values()), return_exceptions=True
-        )
-        POOL.clear()
-        LAST_USED.clear()
-
+        await asyncio.gather(*(c.close() for c in POOL.values()), return_exceptions=True)
+        POOL.clear(); LAST_USED.clear()
 
 async def janitor():
     while True:
@@ -113,7 +56,5 @@ async def janitor():
         async with LOCK:
             for sig, crawler in list(POOL.items()):
                 if now - LAST_USED[sig] > IDLE_TTL:
-                    with suppress(Exception):
-                        await crawler.close()
-                    POOL.pop(sig, None)
-                    LAST_USED.pop(sig, None)
+                    with suppress(Exception): await crawler.close()
+                    POOL.pop(sig, None); LAST_USED.pop(sig, None)

deploy/docker/job.py

@@ -39,7 +39,6 @@ class LlmJobPayload(BaseModel):
     provider: Optional[str] = None
     temperature: Optional[float] = None
     base_url: Optional[str] = None
-    chunking_strategy: Optional[Dict] = None
 
 
 class CrawlJobPayload(BaseModel):
@@ -68,7 +67,6 @@ async def llm_job_enqueue(
         provider=payload.provider,
         temperature=payload.temperature,
         api_base_url=payload.base_url,
-        chunking_strategy_config=payload.chunking_strategy,
     )
 
 

deploy/docker/routers/adaptive.py

@@ -1,270 +0,0 @@
-import uuid
-from typing import Any, Dict
-
-from fastapi import APIRouter, BackgroundTasks, HTTPException
-from schemas import AdaptiveConfigPayload, AdaptiveCrawlRequest, AdaptiveJobStatus
-
-from crawl4ai import AsyncWebCrawler
-from crawl4ai.adaptive_crawler import AdaptiveConfig, AdaptiveCrawler
-from crawl4ai.utils import get_error_context
-
-# --- In-memory storage for job statuses. For production, use Redis or a database. ---
-ADAPTIVE_JOBS: Dict[str, Dict[str, Any]] = {}
-
-# --- APIRouter for Adaptive Crawling Endpoints ---
-router = APIRouter(
-    prefix="/adaptive/digest",
-    tags=["Adaptive Crawling"],
-)
-
-# --- Background Worker Function ---
-
-
-async def run_adaptive_digest(task_id: str, request: AdaptiveCrawlRequest):
-    """The actual async worker that performs the adaptive crawl."""
-    try:
-        # Update job status to RUNNING
-        ADAPTIVE_JOBS[task_id]["status"] = "RUNNING"
-
-        # Create AdaptiveConfig from payload or use default
-        if request.config:
-            adaptive_config = AdaptiveConfig(**request.config.model_dump())
-        else:
-            adaptive_config = AdaptiveConfig()
-
-        # The adaptive crawler needs an instance of the web crawler
-        async with AsyncWebCrawler() as crawler:
-            adaptive_crawler = AdaptiveCrawler(crawler, config=adaptive_config)
-
-            # This is the long-running operation
-            final_state = await adaptive_crawler.digest(
-                start_url=request.start_url, query=request.query
-            )
-
-            # Process the final state into a clean result
-            result_data = {
-                "confidence": final_state.metrics.get("confidence", 0.0),
-                "is_sufficient": adaptive_crawler.is_sufficient,
-                "coverage_stats": adaptive_crawler.coverage_stats,
-                "relevant_content": adaptive_crawler.get_relevant_content(top_k=5),
-            }
-
-            # Update job with the final result
-            ADAPTIVE_JOBS[task_id].update(
-                {
-                    "status": "COMPLETED",
-                    "result": result_data,
-                    "metrics": final_state.metrics,
-                }
-            )
-
-    except Exception as e:
-        # On failure, update the job with an error message
-        import sys
-
-        error_context = get_error_context(sys.exc_info())
-        error_message = f"Adaptive crawl failed: {str(e)}\nContext: {error_context}"
-
-        ADAPTIVE_JOBS[task_id].update({"status": "FAILED", "error": error_message})
-
-
-# --- API Endpoints ---
-
-
-@router.post("/job",
-    summary="Submit Adaptive Crawl Job",
-    description="Start a long-running adaptive crawling job that intelligently discovers relevant content.",
-    response_description="Job ID for status polling",
-    response_model=AdaptiveJobStatus,
-    status_code=202
-)
-async def submit_adaptive_digest_job(
-    request: AdaptiveCrawlRequest,
-    background_tasks: BackgroundTasks,
-):
-    """
-    Submit a new adaptive crawling job.
-    
-    This endpoint starts an intelligent, long-running crawl that automatically
-    discovers and extracts relevant content based on your query. Returns
-    immediately with a task ID for polling.
-    
-    **Request Body:**
-    ```json
-    {
-        "start_url": "https://example.com",
-        "query": "Find all product documentation",
-        "config": {
-            "max_depth": 3,
-            "max_pages": 50,
-            "confidence_threshold": 0.7,
-            "timeout": 300
-        }
-    }
-    ```
-    
-    **Parameters:**
-    - `start_url`: Starting URL for the crawl
-    - `query`: Natural language query describing what to find
-    - `config`: Optional adaptive configuration (max_depth, max_pages, etc.)
-    
-    **Response:**
-    ```json
-    {
-        "task_id": "550e8400-e29b-41d4-a716-446655440000",
-        "status": "PENDING",
-        "metrics": null,
-        "result": null,
-        "error": null
-    }
-    ```
-    
-    **Usage:**
-    ```python
-    # Submit job
-    response = requests.post(
-        "http://localhost:11235/adaptive/digest/job",
-        headers={"Authorization": f"Bearer {token}"},
-        json={
-            "start_url": "https://example.com",
-            "query": "Find all API documentation"
-        }
-    )
-    task_id = response.json()["task_id"]
-    
-    # Poll for results
-    while True:
-        status_response = requests.get(
-            f"http://localhost:11235/adaptive/digest/job/{task_id}",
-            headers={"Authorization": f"Bearer {token}"}
-        )
-        status = status_response.json()
-        if status["status"] in ["COMPLETED", "FAILED"]:
-            print(status["result"])
-            break
-        time.sleep(2)
-    ```
-    
-    **Notes:**
-    - Job runs in background, returns immediately
-    - Use task_id to poll status with GET /adaptive/digest/job/{task_id}
-    - Adaptive crawler intelligently follows links based on relevance
-    - Automatically stops when sufficient content found
-    - Returns HTTP 202 Accepted
-    """
-
-    print("Received adaptive crawl request:", request)
-    task_id = str(uuid.uuid4())
-
-    # Initialize the job in our in-memory store
-    ADAPTIVE_JOBS[task_id] = {
-        "task_id": task_id,
-        "status": "PENDING",
-        "metrics": None,
-        "result": None,
-        "error": None,
-    }
-
-    # Add the long-running task to the background
-    background_tasks.add_task(run_adaptive_digest, task_id, request)
-
-    return ADAPTIVE_JOBS[task_id]
-
-
-@router.get("/job/{task_id}",
-    summary="Get Adaptive Job Status",
-    description="Poll the status and results of an adaptive crawling job.",
-    response_description="Job status, metrics, and results",
-    response_model=AdaptiveJobStatus
-)
-async def get_adaptive_digest_status(task_id: str):
-    """
-    Get the status and result of an adaptive crawling job.
-    
-    Poll this endpoint with the task_id returned from the submission endpoint
-    until the status is 'COMPLETED' or 'FAILED'.
-    
-    **Parameters:**
-    - `task_id`: Job ID from POST /adaptive/digest/job
-    
-    **Response (Running):**
-    ```json
-    {
-        "task_id": "550e8400-e29b-41d4-a716-446655440000",
-        "status": "RUNNING",
-        "metrics": {
-            "confidence": 0.45,
-            "pages_crawled": 15,
-            "relevant_pages": 8
-        },
-        "result": null,
-        "error": null
-    }
-    ```
-    
-    **Response (Completed):**
-    ```json
-    {
-        "task_id": "550e8400-e29b-41d4-a716-446655440000",
-        "status": "COMPLETED",
-        "metrics": {
-            "confidence": 0.85,
-            "pages_crawled": 42,
-            "relevant_pages": 28
-        },
-        "result": {
-            "confidence": 0.85,
-            "is_sufficient": true,
-            "coverage_stats": {...},
-            "relevant_content": [...]
-        },
-        "error": null
-    }
-    ```
-    
-    **Status Values:**
-    - `PENDING`: Job queued, not started yet
-    - `RUNNING`: Job actively crawling
-    - `COMPLETED`: Job finished successfully
-    - `FAILED`: Job encountered an error
-    
-    **Usage:**
-    ```python
-    import time
-    
-    # Poll until complete
-    while True:
-        response = requests.get(
-            f"http://localhost:11235/adaptive/digest/job/{task_id}",
-            headers={"Authorization": f"Bearer {token}"}
-        )
-        job = response.json()
-        
-        print(f"Status: {job['status']}")
-        if job['status'] == 'RUNNING':
-            print(f"Progress: {job['metrics']['pages_crawled']} pages")
-        elif job['status'] == 'COMPLETED':
-            print(f"Found {len(job['result']['relevant_content'])} relevant items")
-            break
-        elif job['status'] == 'FAILED':
-            print(f"Error: {job['error']}")
-            break
-        
-        time.sleep(2)
-    ```
-    
-    **Notes:**
-    - Poll every 1-5 seconds
-    - Metrics updated in real-time while running
-    - Returns 404 if task_id not found
-    - Results include top relevant content and statistics
-    """
-    job = ADAPTIVE_JOBS.get(task_id)
-    if not job:
-        raise HTTPException(status_code=404, detail="Job not found")
-
-    # If the job is running, update the metrics from the live state
-    if job["status"] == "RUNNING" and job.get("live_state"):
-        job["metrics"] = job["live_state"].metrics
-
-    return job

deploy/docker/routers/dispatchers.py

@@ -1,259 +0,0 @@
-"""
-Router for dispatcher management endpoints.
-
-Provides endpoints to:
-- List available dispatchers
-- Get default dispatcher info
-- Get dispatcher statistics
-"""
-
-import logging
-from typing import Dict, List
-
-from fastapi import APIRouter, HTTPException, Request
-from schemas import DispatcherInfo, DispatcherStatsResponse, DispatcherType
-from utils import get_available_dispatchers, get_dispatcher_config
-
-logger = logging.getLogger(__name__)
-
-# --- APIRouter for Dispatcher Endpoints ---
-router = APIRouter(
-    prefix="/dispatchers",
-    tags=["Dispatchers"],
-)
-
-
-@router.get("",
-    summary="List Dispatchers",
-    description="Get information about all available dispatcher types.",
-    response_description="List of dispatcher configurations and features",
-    response_model=List[DispatcherInfo]
-)
-async def list_dispatchers(request: Request):
-    """
-    List all available dispatcher types.
-    
-    Returns information about each dispatcher type including name, description,
-    configuration parameters, and key features.
-    
-    **Dispatchers:**
-    - `memory_adaptive`: Automatically manages crawler instances based on memory
-    - `semaphore`: Simple semaphore-based concurrency control
-    
-    **Response:**
-    ```json
-    [
-        {
-            "type": "memory_adaptive",
-            "name": "Memory Adaptive Dispatcher",
-            "description": "Automatically adjusts crawler pool based on memory usage",
-            "config": {...},
-            "features": ["Auto-scaling", "Memory monitoring", "Smart throttling"]
-        },
-        {
-            "type": "semaphore",
-            "name": "Semaphore Dispatcher",
-            "description": "Simple semaphore-based concurrency control",
-            "config": {...},
-            "features": ["Fixed concurrency", "Simple queue"]
-        }
-    ]
-    ```
-    
-    **Usage:**
-    ```python
-    response = requests.get(
-        "http://localhost:11235/dispatchers",
-        headers={"Authorization": f"Bearer {token}"}
-    )
-    dispatchers = response.json()
-    for dispatcher in dispatchers:
-        print(f"{dispatcher['type']}: {dispatcher['description']}")
-    ```
-    
-    **Notes:**
-    - Lists all registered dispatcher types
-    - Shows configuration options for each
-    - Use with /crawl endpoint's `dispatcher` parameter
-    """
-    try:
-        dispatchers_info = get_available_dispatchers()
-        
-        result = []
-        for dispatcher_type, info in dispatchers_info.items():
-            result.append(
-                DispatcherInfo(
-                    type=DispatcherType(dispatcher_type),
-                    name=info["name"],
-                    description=info["description"],
-                    config=info["config"],
-                    features=info["features"],
-                )
-            )
-        
-        return result
-    except Exception as e:
-        logger.error(f"Error listing dispatchers: {e}")
-        raise HTTPException(status_code=500, detail=f"Failed to list dispatchers: {str(e)}")
-
-
-@router.get("/default",
-    summary="Get Default Dispatcher",
-    description="Get information about the currently configured default dispatcher.",
-    response_description="Default dispatcher information",
-    response_model=Dict
-)
-async def get_default_dispatcher(request: Request):
-    """
-    Get information about the current default dispatcher.
-    
-    Returns the dispatcher type, configuration, and status for the default
-    dispatcher used when no specific dispatcher is requested.
-    
-    **Response:**
-    ```json
-    {
-        "type": "memory_adaptive",
-        "config": {
-            "max_memory_percent": 80,
-            "check_interval": 10,
-            "min_instances": 1,
-            "max_instances": 10
-        },
-        "active": true
-    }
-    ```
-    
-    **Usage:**
-    ```python
-    response = requests.get(
-        "http://localhost:11235/dispatchers/default",
-        headers={"Authorization": f"Bearer {token}"}
-    )
-    default_dispatcher = response.json()
-    print(f"Default: {default_dispatcher['type']}")
-    ```
-    
-    **Notes:**
-    - Shows which dispatcher is used by default
-    - Default can be configured via server settings
-    - Override with `dispatcher` parameter in /crawl requests
-    """
-    try:
-        default_type = request.app.state.default_dispatcher_type
-        dispatcher = request.app.state.dispatchers.get(default_type)
-        
-        if not dispatcher:
-            raise HTTPException(
-                status_code=500, 
-                detail=f"Default dispatcher '{default_type}' not initialized"
-            )
-        
-        return {
-            "type": default_type,
-            "config": get_dispatcher_config(default_type),
-            "active": True,
-        }
-    except HTTPException:
-        raise
-    except Exception as e:
-        logger.error(f"Error getting default dispatcher: {e}")
-        raise HTTPException(
-            status_code=500, 
-            detail=f"Failed to get default dispatcher: {str(e)}"
-        )
-
-
-@router.get("/{dispatcher_type}/stats",
-    summary="Get Dispatcher Statistics",
-    description="Get runtime statistics for a specific dispatcher.",
-    response_description="Dispatcher statistics and metrics",
-    response_model=DispatcherStatsResponse
-)
-async def get_dispatcher_stats(dispatcher_type: DispatcherType, request: Request):
-    """
-    Get runtime statistics for a specific dispatcher.
-    
-    Returns active sessions, configuration, and dispatcher-specific metrics.
-    Useful for monitoring and debugging dispatcher performance.
-    
-    **Parameters:**
-    - `dispatcher_type`: Dispatcher type (memory_adaptive, semaphore)
-    
-    **Response:**
-    ```json
-    {
-        "type": "memory_adaptive",
-        "active_sessions": 3,
-        "config": {
-            "max_memory_percent": 80,
-            "check_interval": 10
-        },
-        "stats": {
-            "current_memory_percent": 45.2,
-            "active_instances": 3,
-            "max_instances": 10,
-            "throttled_count": 0
-        }
-    }
-    ```
-    
-    **Usage:**
-    ```python
-    response = requests.get(
-        "http://localhost:11235/dispatchers/memory_adaptive/stats",
-        headers={"Authorization": f"Bearer {token}"}
-    )
-    stats = response.json()
-    print(f"Active sessions: {stats['active_sessions']}")
-    print(f"Memory usage: {stats['stats']['current_memory_percent']}%")
-    ```
-    
-    **Notes:**
-    - Real-time statistics
-    - Stats vary by dispatcher type
-    - Use for monitoring and capacity planning
-    - Returns 404 if dispatcher type not found
-    """
-    try:
-        dispatcher_name = dispatcher_type.value
-        dispatcher = request.app.state.dispatchers.get(dispatcher_name)
-        
-        if not dispatcher:
-            raise HTTPException(
-                status_code=404, 
-                detail=f"Dispatcher '{dispatcher_name}' not found or not initialized"
-            )
-        
-        # Get basic stats
-        stats = {
-            "type": dispatcher_type,
-            "active_sessions": dispatcher.concurrent_sessions,
-            "config": get_dispatcher_config(dispatcher_name),
-            "stats": {}
-        }
-        
-        # Add dispatcher-specific stats
-        if dispatcher_name == "memory_adaptive":
-            stats["stats"] = {
-                "current_memory_percent": getattr(dispatcher, "current_memory_percent", 0.0),
-                "memory_pressure_mode": getattr(dispatcher, "memory_pressure_mode", False),
-                "task_queue_size": dispatcher.task_queue.qsize() if hasattr(dispatcher, "task_queue") else 0,
-            }
-        elif dispatcher_name == "semaphore":
-            # For semaphore dispatcher, show semaphore availability
-            if hasattr(dispatcher, "semaphore_count"):
-                stats["stats"] = {
-                    "max_concurrent": dispatcher.semaphore_count,
-                }
-        
-        return DispatcherStatsResponse(**stats)
-        
-    except HTTPException:
-        raise
-    except Exception as e:
-        logger.error(f"Error getting dispatcher stats for '{dispatcher_type}': {e}")
-        raise HTTPException(
-            status_code=500, 
-            detail=f"Failed to get dispatcher stats: {str(e)}"
-        )

deploy/docker/routers/monitoring.py

@@ -1,746 +0,0 @@
-"""
-Monitoring and Profiling Router
-
-Provides endpoints for:
-- Browser performance profiling
-- Real-time crawler statistics
-- System resource monitoring
-- Session management
-"""
-
-from fastapi import APIRouter, HTTPException, BackgroundTasks, Query
-from fastapi.responses import StreamingResponse
-from pydantic import BaseModel, Field
-from typing import Dict, List, Optional, Any, AsyncGenerator
-from datetime import datetime, timedelta
-import uuid
-import asyncio
-import json
-import time
-import psutil
-import logging
-from collections import defaultdict
-
-logger = logging.getLogger(__name__)
-
-router = APIRouter(
-    prefix="/monitoring",
-    tags=["Monitoring & Profiling"],
-    responses={
-        404: {"description": "Session not found"},
-        500: {"description": "Internal server error"}
-    }
-)
-
-# ============================================================================
-# Data Structures
-# ============================================================================
-
-# In-memory storage for profiling sessions
-PROFILING_SESSIONS: Dict[str, Dict[str, Any]] = {}
-
-# Real-time crawler statistics
-CRAWLER_STATS = {
-    "active_crawls": 0,
-    "total_crawls": 0,
-    "successful_crawls": 0,
-    "failed_crawls": 0,
-    "total_bytes_processed": 0,
-    "average_response_time_ms": 0.0,
-    "last_updated": datetime.now().isoformat(),
-}
-
-# Per-URL statistics
-URL_STATS: Dict[str, Dict[str, Any]] = defaultdict(lambda: {
-    "total_requests": 0,
-    "success_count": 0,
-    "failure_count": 0,
-    "average_time_ms": 0.0,
-    "last_accessed": None,
-})
-
-
-# ============================================================================
-# Pydantic Models
-# ============================================================================
-
-class ProfilingStartRequest(BaseModel):
-    """Request to start a profiling session."""
-    url: str = Field(..., description="URL to profile")
-    browser_config: Optional[Dict[str, Any]] = Field(
-        default_factory=dict,
-        description="Browser configuration"
-    )
-    crawler_config: Optional[Dict[str, Any]] = Field(
-        default_factory=dict,
-        description="Crawler configuration"
-    )
-    profile_duration: Optional[int] = Field(
-        default=30,
-        ge=5,
-        le=300,
-        description="Maximum profiling duration in seconds"
-    )
-    collect_network: bool = Field(
-        default=True,
-        description="Collect network performance data"
-    )
-    collect_memory: bool = Field(
-        default=True,
-        description="Collect memory usage data"
-    )
-    collect_cpu: bool = Field(
-        default=True,
-        description="Collect CPU usage data"
-    )
-    
-    class Config:
-        schema_extra = {
-            "example": {
-                "url": "https://example.com",
-                "profile_duration": 30,
-                "collect_network": True,
-                "collect_memory": True,
-                "collect_cpu": True
-            }
-        }
-
-
-class ProfilingSession(BaseModel):
-    """Profiling session information."""
-    session_id: str = Field(..., description="Unique session identifier")
-    status: str = Field(..., description="Session status: running, completed, failed")
-    url: str = Field(..., description="URL being profiled")
-    start_time: str = Field(..., description="Session start time (ISO format)")
-    end_time: Optional[str] = Field(None, description="Session end time (ISO format)")
-    duration_seconds: Optional[float] = Field(None, description="Total duration in seconds")
-    results: Optional[Dict[str, Any]] = Field(None, description="Profiling results")
-    error: Optional[str] = Field(None, description="Error message if failed")
-    
-    class Config:
-        schema_extra = {
-            "example": {
-                "session_id": "abc123",
-                "status": "completed",
-                "url": "https://example.com",
-                "start_time": "2025-10-16T10:30:00",
-                "end_time": "2025-10-16T10:30:30",
-                "duration_seconds": 30.5,
-                "results": {
-                    "performance": {
-                        "page_load_time_ms": 1234,
-                        "dom_content_loaded_ms": 890,
-                        "first_paint_ms": 567
-                    }
-                }
-            }
-        }
-
-
-class CrawlerStats(BaseModel):
-    """Current crawler statistics."""
-    active_crawls: int = Field(..., description="Number of currently active crawls")
-    total_crawls: int = Field(..., description="Total crawls since server start")
-    successful_crawls: int = Field(..., description="Number of successful crawls")
-    failed_crawls: int = Field(..., description="Number of failed crawls")
-    success_rate: float = Field(..., description="Success rate percentage")
-    total_bytes_processed: int = Field(..., description="Total bytes processed")
-    average_response_time_ms: float = Field(..., description="Average response time")
-    uptime_seconds: float = Field(..., description="Server uptime in seconds")
-    memory_usage_mb: float = Field(..., description="Current memory usage in MB")
-    cpu_percent: float = Field(..., description="Current CPU usage percentage")
-    last_updated: str = Field(..., description="Last update timestamp")
-
-
-class URLStatistics(BaseModel):
-    """Statistics for a specific URL pattern."""
-    url_pattern: str
-    total_requests: int
-    success_count: int
-    failure_count: int
-    success_rate: float
-    average_time_ms: float
-    last_accessed: Optional[str]
-
-
-class SessionListResponse(BaseModel):
-    """List of profiling sessions."""
-    total: int
-    sessions: List[ProfilingSession]
-
-
-# ============================================================================
-# Helper Functions
-# ============================================================================
-
-def get_system_stats() -> Dict[str, Any]:
-    """Get current system resource usage."""
-    try:
-        process = psutil.Process()
-        
-        return {
-            "memory_usage_mb": process.memory_info().rss / 1024 / 1024,
-            "cpu_percent": process.cpu_percent(interval=0.1),
-            "num_threads": process.num_threads(),
-            "open_files": len(process.open_files()),
-            "connections": len(process.connections()),
-        }
-    except Exception as e:
-        logger.error(f"Error getting system stats: {e}")
-        return {
-            "memory_usage_mb": 0.0,
-            "cpu_percent": 0.0,
-            "num_threads": 0,
-            "open_files": 0,
-            "connections": 0,
-        }
-
-
-def cleanup_old_sessions(max_age_hours: int = 24):
-    """Remove old profiling sessions to prevent memory leaks."""
-    cutoff = datetime.now() - timedelta(hours=max_age_hours)
-    
-    to_remove = []
-    for session_id, session in PROFILING_SESSIONS.items():
-        try:
-            start_time = datetime.fromisoformat(session["start_time"])
-            if start_time < cutoff:
-                to_remove.append(session_id)
-        except (ValueError, KeyError):
-            continue
-    
-    for session_id in to_remove:
-        del PROFILING_SESSIONS[session_id]
-        logger.info(f"Cleaned up old session: {session_id}")
-    
-    return len(to_remove)
-
-
-# ============================================================================
-# Profiling Endpoints
-# ============================================================================
-
-@router.post(
-    "/profile/start",
-    response_model=ProfilingSession,
-    summary="Start profiling session",
-    description="Start a new browser profiling session for performance analysis"
-)
-async def start_profiling_session(
-    request: ProfilingStartRequest,
-    background_tasks: BackgroundTasks
-):
-    """
-    Start a new profiling session.
-    
-    Returns a session ID that can be used to retrieve results later.
-    The profiling runs in the background and collects:
-    - Page load performance metrics
-    - Network requests and timing
-    - Memory usage patterns
-    - CPU utilization
-    - Browser-specific metrics
-    """
-    session_id = str(uuid.uuid4())
-    start_time = datetime.now()
-    
-    session_data = {
-        "session_id": session_id,
-        "status": "running",
-        "url": request.url,
-        "start_time": start_time.isoformat(),
-        "end_time": None,
-        "duration_seconds": None,
-        "results": None,
-        "error": None,
-        "config": {
-            "profile_duration": request.profile_duration,
-            "collect_network": request.collect_network,
-            "collect_memory": request.collect_memory,
-            "collect_cpu": request.collect_cpu,
-        }
-    }
-    
-    PROFILING_SESSIONS[session_id] = session_data
-    
-    # Add background task to run profiling
-    background_tasks.add_task(
-        run_profiling_session,
-        session_id,
-        request
-    )
-    
-    logger.info(f"Started profiling session {session_id} for {request.url}")
-    
-    return ProfilingSession(**session_data)
-
-
-@router.get(
-    "/profile/{session_id}",
-    response_model=ProfilingSession,
-    summary="Get profiling results",
-    description="Retrieve results from a profiling session"
-)
-async def get_profiling_results(session_id: str):
-    """
-    Get profiling session results.
-    
-    Returns the current status and results of a profiling session.
-    If the session is still running, results will be None.
-    """
-    if session_id not in PROFILING_SESSIONS:
-        raise HTTPException(
-            status_code=404,
-            detail=f"Profiling session '{session_id}' not found"
-        )
-    
-    session = PROFILING_SESSIONS[session_id]
-    return ProfilingSession(**session)
-
-
-@router.get(
-    "/profile",
-    response_model=SessionListResponse,
-    summary="List profiling sessions",
-    description="List all profiling sessions with optional filtering"
-)
-async def list_profiling_sessions(
-    status: Optional[str] = Query(None, description="Filter by status: running, completed, failed"),
-    limit: int = Query(50, ge=1, le=500, description="Maximum number of sessions to return")
-):
-    """
-    List all profiling sessions.
-    
-    Can be filtered by status and limited in number.
-    """
-    sessions = list(PROFILING_SESSIONS.values())
-    
-    # Filter by status if provided
-    if status:
-        sessions = [s for s in sessions if s["status"] == status]
-    
-    # Sort by start time (newest first)
-    sessions.sort(key=lambda x: x["start_time"], reverse=True)
-    
-    # Limit results
-    sessions = sessions[:limit]
-    
-    return SessionListResponse(
-        total=len(sessions),
-        sessions=[ProfilingSession(**s) for s in sessions]
-    )
-
-
-@router.delete(
-    "/profile/{session_id}",
-    summary="Delete profiling session",
-    description="Delete a profiling session and its results"
-)
-async def delete_profiling_session(session_id: str):
-    """
-    Delete a profiling session.
-    
-    Removes the session and all associated data from memory.
-    """
-    if session_id not in PROFILING_SESSIONS:
-        raise HTTPException(
-            status_code=404,
-            detail=f"Profiling session '{session_id}' not found"
-        )
-    
-    session = PROFILING_SESSIONS.pop(session_id)
-    logger.info(f"Deleted profiling session {session_id}")
-    
-    return {
-        "success": True,
-        "message": f"Session {session_id} deleted",
-        "session": ProfilingSession(**session)
-    }
-
-
-@router.post(
-    "/profile/cleanup",
-    summary="Cleanup old sessions",
-    description="Remove old profiling sessions to free memory"
-)
-async def cleanup_sessions(
-    max_age_hours: int = Query(24, ge=1, le=168, description="Maximum age in hours")
-):
-    """
-    Cleanup old profiling sessions.
-    
-    Removes sessions older than the specified age.
-    """
-    removed = cleanup_old_sessions(max_age_hours)
-    
-    return {
-        "success": True,
-        "removed_count": removed,
-        "remaining_count": len(PROFILING_SESSIONS),
-        "message": f"Removed {removed} sessions older than {max_age_hours} hours"
-    }
-
-
-# ============================================================================
-# Statistics Endpoints
-# ============================================================================
-
-@router.get(
-    "/stats",
-    response_model=CrawlerStats,
-    summary="Get crawler statistics",
-    description="Get current crawler statistics and system metrics"
-)
-async def get_crawler_stats():
-    """
-    Get current crawler statistics.
-    
-    Returns real-time metrics about:
-    - Active and total crawls
-    - Success/failure rates
-    - Response times
-    - System resource usage
-    """
-    system_stats = get_system_stats()
-    
-    total = CRAWLER_STATS["successful_crawls"] + CRAWLER_STATS["failed_crawls"]
-    success_rate = (
-        (CRAWLER_STATS["successful_crawls"] / total * 100)
-        if total > 0 else 0.0
-    )
-    
-    # Calculate uptime
-    # In a real implementation, you'd track server start time
-    uptime_seconds = 0.0  # Placeholder
-    
-    stats = CrawlerStats(
-        active_crawls=CRAWLER_STATS["active_crawls"],
-        total_crawls=CRAWLER_STATS["total_crawls"],
-        successful_crawls=CRAWLER_STATS["successful_crawls"],
-        failed_crawls=CRAWLER_STATS["failed_crawls"],
-        success_rate=success_rate,
-        total_bytes_processed=CRAWLER_STATS["total_bytes_processed"],
-        average_response_time_ms=CRAWLER_STATS["average_response_time_ms"],
-        uptime_seconds=uptime_seconds,
-        memory_usage_mb=system_stats["memory_usage_mb"],
-        cpu_percent=system_stats["cpu_percent"],
-        last_updated=datetime.now().isoformat()
-    )
-    
-    return stats
-
-
-@router.get(
-    "/stats/stream",
-    summary="Stream crawler statistics",
-    description="Server-Sent Events stream of real-time crawler statistics"
-)
-async def stream_crawler_stats(
-    interval: int = Query(2, ge=1, le=60, description="Update interval in seconds")
-):
-    """
-    Stream real-time crawler statistics.
-    
-    Returns an SSE (Server-Sent Events) stream that pushes
-    statistics updates at the specified interval.
-    
-    Example:
-        ```javascript
-        const eventSource = new EventSource('/monitoring/stats/stream?interval=2');
-        eventSource.onmessage = (event) => {
-            const stats = JSON.parse(event.data);
-            console.log('Stats:', stats);
-        };
-        ```
-    """
-    
-    async def generate_stats() -> AsyncGenerator[str, None]:
-        """Generate stats stream."""
-        try:
-            while True:
-                # Get current stats
-                stats = await get_crawler_stats()
-                
-                # Format as SSE
-                data = json.dumps(stats.dict())
-                yield f"data: {data}\n\n"
-                
-                # Wait for next interval
-                await asyncio.sleep(interval)
-                
-        except asyncio.CancelledError:
-            logger.info("Stats stream cancelled by client")
-        except Exception as e:
-            logger.error(f"Error in stats stream: {e}")
-            yield f"event: error\ndata: {json.dumps({'error': str(e)})}\n\n"
-    
-    return StreamingResponse(
-        generate_stats(),
-        media_type="text/event-stream",
-        headers={
-            "Cache-Control": "no-cache",
-            "Connection": "keep-alive",
-            "X-Accel-Buffering": "no",
-        }
-    )
-
-
-@router.get(
-    "/stats/urls",
-    response_model=List[URLStatistics],
-    summary="Get URL statistics",
-    description="Get statistics for crawled URLs"
-)
-async def get_url_statistics(
-    limit: int = Query(100, ge=1, le=1000, description="Maximum number of URLs to return"),
-    sort_by: str = Query("total_requests", description="Sort field: total_requests, success_rate, average_time_ms")
-):
-    """
-    Get statistics for crawled URLs.
-    
-    Returns metrics for each URL that has been crawled,
-    including request counts, success rates, and timing.
-    """
-    stats_list = []
-    
-    for url, stats in URL_STATS.items():
-        total = stats["total_requests"]
-        success_rate = (stats["success_count"] / total * 100) if total > 0 else 0.0
-        
-        stats_list.append(URLStatistics(
-            url_pattern=url,
-            total_requests=stats["total_requests"],
-            success_count=stats["success_count"],
-            failure_count=stats["failure_count"],
-            success_rate=success_rate,
-            average_time_ms=stats["average_time_ms"],
-            last_accessed=stats["last_accessed"]
-        ))
-    
-    # Sort
-    if sort_by == "success_rate":
-        stats_list.sort(key=lambda x: x.success_rate, reverse=True)
-    elif sort_by == "average_time_ms":
-        stats_list.sort(key=lambda x: x.average_time_ms)
-    else:  # total_requests
-        stats_list.sort(key=lambda x: x.total_requests, reverse=True)
-    
-    return stats_list[:limit]
-
-
-@router.post(
-    "/stats/reset",
-    summary="Reset statistics",
-    description="Reset all crawler statistics to zero"
-)
-async def reset_statistics():
-    """
-    Reset all statistics.
-    
-    Clears all accumulated statistics but keeps the server running.
-    Useful for testing or starting fresh measurements.
-    """
-    global CRAWLER_STATS, URL_STATS
-    
-    CRAWLER_STATS = {
-        "active_crawls": 0,
-        "total_crawls": 0,
-        "successful_crawls": 0,
-        "failed_crawls": 0,
-        "total_bytes_processed": 0,
-        "average_response_time_ms": 0.0,
-        "last_updated": datetime.now().isoformat(),
-    }
-    
-    URL_STATS.clear()
-    
-    logger.info("All statistics reset")
-    
-    return {
-        "success": True,
-        "message": "All statistics have been reset",
-        "timestamp": datetime.now().isoformat()
-    }
-
-
-# ============================================================================
-# Background Tasks
-# ============================================================================
-
-async def run_profiling_session(session_id: str, request: ProfilingStartRequest):
-    """
-    Background task to run profiling session.
-    
-    This performs the actual profiling work:
-    1. Creates a crawler with profiling enabled
-    2. Crawls the target URL
-    3. Collects performance metrics
-    4. Stores results in the session
-    """
-    start_time = time.time()
-    
-    try:
-        from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
-        from crawl4ai.browser_profiler import BrowserProfiler
-        
-        logger.info(f"Starting profiling for session {session_id}")
-        
-        # Create profiler
-        profiler = BrowserProfiler()
-        
-        # Configure browser and crawler
-        browser_config = BrowserConfig.load(request.browser_config)
-        crawler_config = CrawlerRunConfig.load(request.crawler_config)
-        
-        # Enable profiling options
-        browser_config.profiling_enabled = True
-        
-        results = {}
-        
-        async with AsyncWebCrawler(config=browser_config) as crawler:
-            # Start profiling
-            profiler.start()
-            
-            # Collect system stats before
-            stats_before = get_system_stats()
-            
-            # Crawl with timeout
-            try:
-                result = await asyncio.wait_for(
-                    crawler.arun(request.url, config=crawler_config),
-                    timeout=request.profile_duration
-                )
-                
-                crawl_success = result.success
-                
-            except asyncio.TimeoutError:
-                logger.warning(f"Profiling session {session_id} timed out")
-                crawl_success = False
-                result = None
-            
-            # Stop profiling
-            profiler_results = profiler.stop()
-            
-            # Collect system stats after
-            stats_after = get_system_stats()
-            
-            # Build results
-            results = {
-                "crawl_success": crawl_success,
-                "url": request.url,
-                "performance": profiler_results if profiler_results else {},
-                "system": {
-                    "before": stats_before,
-                    "after": stats_after,
-                    "delta": {
-                        "memory_mb": stats_after["memory_usage_mb"] - stats_before["memory_usage_mb"],
-                        "cpu_percent": stats_after["cpu_percent"] - stats_before["cpu_percent"],
-                    }
-                }
-            }
-            
-            if result:
-                results["content"] = {
-                    "markdown_length": len(result.markdown) if result.markdown else 0,
-                    "html_length": len(result.html) if result.html else 0,
-                    "links_count": len(result.links["internal"]) + len(result.links["external"]),
-                    "media_count": len(result.media["images"]) + len(result.media["videos"]),
-                }
-        
-        # Update session with results
-        end_time = time.time()
-        duration = end_time - start_time
-        
-        PROFILING_SESSIONS[session_id].update({
-            "status": "completed",
-            "end_time": datetime.now().isoformat(),
-            "duration_seconds": duration,
-            "results": results
-        })
-        
-        logger.info(f"Profiling session {session_id} completed in {duration:.2f}s")
-        
-    except Exception as e:
-        logger.error(f"Profiling session {session_id} failed: {str(e)}")
-        
-        PROFILING_SESSIONS[session_id].update({
-            "status": "failed",
-            "end_time": datetime.now().isoformat(),
-            "duration_seconds": time.time() - start_time,
-            "error": str(e)
-        })
-
-
-# ============================================================================
-# Middleware Integration Points
-# ============================================================================
-
-def track_crawl_start():
-    """Call this when a crawl starts."""
-    CRAWLER_STATS["active_crawls"] += 1
-    CRAWLER_STATS["total_crawls"] += 1
-    CRAWLER_STATS["last_updated"] = datetime.now().isoformat()
-
-
-def track_crawl_end(url: str, success: bool, duration_ms: float, bytes_processed: int = 0):
-    """Call this when a crawl ends."""
-    CRAWLER_STATS["active_crawls"] = max(0, CRAWLER_STATS["active_crawls"] - 1)
-    
-    if success:
-        CRAWLER_STATS["successful_crawls"] += 1
-    else:
-        CRAWLER_STATS["failed_crawls"] += 1
-    
-    CRAWLER_STATS["total_bytes_processed"] += bytes_processed
-    
-    # Update average response time (running average)
-    total = CRAWLER_STATS["successful_crawls"] + CRAWLER_STATS["failed_crawls"]
-    current_avg = CRAWLER_STATS["average_response_time_ms"]
-    CRAWLER_STATS["average_response_time_ms"] = (
-        (current_avg * (total - 1) + duration_ms) / total
-    )
-    
-    # Update URL stats
-    url_stat = URL_STATS[url]
-    url_stat["total_requests"] += 1
-    
-    if success:
-        url_stat["success_count"] += 1
-    else:
-        url_stat["failure_count"] += 1
-    
-    # Update average time for this URL
-    total_url = url_stat["total_requests"]
-    current_avg_url = url_stat["average_time_ms"]
-    url_stat["average_time_ms"] = (
-        (current_avg_url * (total_url - 1) + duration_ms) / total_url
-    )
-    url_stat["last_accessed"] = datetime.now().isoformat()
-    
-    CRAWLER_STATS["last_updated"] = datetime.now().isoformat()
-
-
-# ============================================================================
-# Health Check
-# ============================================================================
-
-@router.get(
-    "/health",
-    summary="Health check",
-    description="Check if monitoring system is operational"
-)
-async def health_check():
-    """
-    Health check endpoint.
-    
-    Returns status of the monitoring system.
-    """
-    system_stats = get_system_stats()
-    
-    return {
-        "status": "healthy",
-        "timestamp": datetime.now().isoformat(),
-        "active_sessions": len([s for s in PROFILING_SESSIONS.values() if s["status"] == "running"]),
-        "total_sessions": len(PROFILING_SESSIONS),
-        "system": system_stats
-    }

deploy/docker/routers/scripts.py

@@ -1,306 +0,0 @@
-from typing import Optional
-
-from fastapi import APIRouter, File, Form, HTTPException, UploadFile
-from schemas import C4AScriptPayload
-
-from crawl4ai.script import (
-    CompilationResult,
-    ValidationResult,
-    # ErrorDetail
-)
-
-# Import all necessary components from the crawl4ai library
-# C4A Script Language Support
-from crawl4ai.script import (
-    compile as c4a_compile,
-)
-from crawl4ai.script import (
-    validate as c4a_validate,
-)
-
-# --- APIRouter for c4a Scripts Endpoints ---
-router = APIRouter(
-    prefix="/c4a",
-    tags=["c4a Scripts"],
-)
-
-# --- Background Worker Function ---
-
-
-@router.post("/validate",
-    summary="Validate C4A-Script",
-    description="Validate the syntax of a C4A-Script without compiling it.",
-    response_description="Validation result with errors if any",
-    response_model=ValidationResult
-)
-async def validate_c4a_script_endpoint(payload: C4AScriptPayload):
-    """
-    Validate the syntax of a C4A-Script.
-    
-    Checks the script syntax without compiling to executable JavaScript.
-    Returns detailed error information if validation fails.
-    
-    **Request Body:**
-    ```json
-    {
-        "script": "NAVIGATE https://example.com\\nWAIT 2\\nCLICK button.submit"
-    }
-    ```
-    
-    **Response (Valid):**
-    ```json
-    {
-        "success": true,
-        "errors": []
-    }
-    ```
-    
-    **Response (Invalid):**
-    ```json
-    {
-        "success": false,
-        "errors": [
-            {
-                "line": 3,
-                "message": "Unknown command: CLCK",
-                "type": "SyntaxError"
-            }
-        ]
-    }
-    ```
-    
-    **Usage:**
-    ```python
-    response = requests.post(
-        "http://localhost:11235/c4a/validate",
-        headers={"Authorization": f"Bearer {token}"},
-        json={
-            "script": "NAVIGATE https://example.com\\nWAIT 2"
-        }
-    )
-    result = response.json()
-    if result["success"]:
-        print("Script is valid!")
-    else:
-        for error in result["errors"]:
-            print(f"Line {error['line']}: {error['message']}")
-    ```
-    
-    **Notes:**
-    - Validates syntax only, doesn't execute
-    - Returns detailed error locations
-    - Use before compiling to check for issues
-    """
-    # The validate function is designed not to raise exceptions
-    validation_result = c4a_validate(payload.script)
-    return validation_result
-
-
-@router.post("/compile",
-    summary="Compile C4A-Script",
-    description="Compile a C4A-Script into executable JavaScript code.",
-    response_description="Compiled JavaScript code or compilation errors",
-    response_model=CompilationResult
-)
-async def compile_c4a_script_endpoint(payload: C4AScriptPayload):
-    """
-    Compile a C4A-Script into executable JavaScript.
-    
-    Transforms high-level C4A-Script commands into JavaScript that can be
-    executed in a browser context.
-    
-    **Request Body:**
-    ```json
-    {
-        "script": "NAVIGATE https://example.com\\nWAIT 2\\nCLICK button.submit"
-    }
-    ```
-    
-    **Response (Success):**
-    ```json
-    {
-        "success": true,
-        "javascript": "await page.goto('https://example.com');\\nawait page.waitForTimeout(2000);\\nawait page.click('button.submit');",
-        "errors": []
-    }
-    ```
-    
-    **Response (Error):**
-    ```json
-    {
-        "success": false,
-        "javascript": null,
-        "errors": [
-            {
-                "line": 2,
-                "message": "Invalid WAIT duration",
-                "type": "CompilationError"
-            }
-        ]
-    }
-    ```
-    
-    **Usage:**
-    ```python
-    response = requests.post(
-        "http://localhost:11235/c4a/compile",
-        headers={"Authorization": f"Bearer {token}"},
-        json={
-            "script": "NAVIGATE https://example.com\\nCLICK .login-button"
-        }
-    )
-    result = response.json()
-    if result["success"]:
-        print("Compiled JavaScript:")
-        print(result["javascript"])
-    else:
-        print("Compilation failed:", result["errors"])
-    ```
-    
-    **C4A-Script Commands:**
-    - `NAVIGATE <url>` - Navigate to URL
-    - `WAIT <seconds>` - Wait for specified time
-    - `CLICK <selector>` - Click element
-    - `TYPE <selector> <text>` - Type text into element
-    - `SCROLL <direction>` - Scroll page
-    - And many more...
-    
-    **Notes:**
-    - Returns HTTP 400 if compilation fails
-    - JavaScript can be used with /execute_js endpoint
-    - Simplifies browser automation scripting
-    """
-    # The compile function also returns a result object instead of raising
-    compilation_result = c4a_compile(payload.script)
-
-    if not compilation_result.success:
-        # You can optionally raise an HTTP exception for failed compilations
-        # This makes it clearer on the client-side that it was a bad request
-        raise HTTPException(
-            status_code=400,
-            detail=compilation_result.to_dict(),  # FastAPI will serialize this
-        )
-
-    return compilation_result
-
-
-@router.post("/compile-file",
-    summary="Compile C4A-Script from File",
-    description="Compile a C4A-Script from an uploaded file or form string.",
-    response_description="Compiled JavaScript code or compilation errors",
-    response_model=CompilationResult
-)
-async def compile_c4a_script_file_endpoint(
-    file: Optional[UploadFile] = File(None), script: Optional[str] = Form(None)
-):
-    """
-    Compile a C4A-Script from file upload or form data.
-    
-    Accepts either a file upload or a string parameter. Useful for uploading
-    C4A-Script files or sending multipart form data.
-    
-    **Parameters:**
-    - `file`: C4A-Script file upload (multipart/form-data)
-    - `script`: C4A-Script content as string (form field)
-    
-    **Note:** Provide either file OR script, not both.
-    
-    **Request (File Upload):**
-    ```bash
-    curl -X POST "http://localhost:11235/c4a/compile-file" \\
-      -H "Authorization: Bearer YOUR_TOKEN" \\
-      -F "file=@myscript.c4a"
-    ```
-    
-    **Request (Form String):**
-    ```bash
-    curl -X POST "http://localhost:11235/c4a/compile-file" \\
-      -H "Authorization: Bearer YOUR_TOKEN" \\
-      -F "script=NAVIGATE https://example.com"
-    ```
-    
-    **Response:**
-    ```json
-    {
-        "success": true,
-        "javascript": "await page.goto('https://example.com');",
-        "errors": []
-    }
-    ```
-    
-    **Usage (Python with file):**
-    ```python
-    with open('script.c4a', 'rb') as f:
-        response = requests.post(
-            "http://localhost:11235/c4a/compile-file",
-            headers={"Authorization": f"Bearer {token}"},
-            files={"file": f}
-        )
-    result = response.json()
-    print(result["javascript"])
-    ```
-    
-    **Usage (Python with string):**
-    ```python
-    response = requests.post(
-        "http://localhost:11235/c4a/compile-file",
-        headers={"Authorization": f"Bearer {token}"},
-        data={"script": "NAVIGATE https://example.com"}
-    )
-    result = response.json()
-    print(result["javascript"])
-    ```
-    
-    **Notes:**
-    - File must be UTF-8 encoded text
-    - Use for batch script compilation
-    - Returns HTTP 400 if both or neither parameter provided
-    - Returns HTTP 400 if compilation fails
-    """
-    script_content = None
-
-    # Validate that at least one input is provided
-    if not file and not script:
-        raise HTTPException(
-            status_code=400,
-            detail={"error": "Either 'file' or 'script' parameter must be provided"},
-        )
-
-    # If both are provided, prioritize the file
-    if file and script:
-        raise HTTPException(
-            status_code=400,
-            detail={"error": "Please provide either 'file' or 'script', not both"},
-        )
-
-    # Handle file upload
-    if file:
-        try:
-            file_content = await file.read()
-            script_content = file_content.decode("utf-8")
-        except UnicodeDecodeError as exc:
-            raise HTTPException(
-                status_code=400,
-                detail={"error": "File must be a valid UTF-8 text file"},
-            ) from exc
-        except Exception as e:
-            raise HTTPException(
-                status_code=400, detail={"error": f"Error reading file: {str(e)}"}
-            ) from e
-
-    # Handle string content
-    elif script:
-        script_content = script
-
-    # Compile the script content
-    compilation_result = c4a_compile(script_content)
-
-    if not compilation_result.success:
-        # You can optionally raise an HTTP exception for failed compilations
-        # This makes it clearer on the client-side that it was a bad request
-        raise HTTPException(
-            status_code=400,
-            detail=compilation_result.to_dict(),  # FastAPI will serialize this
-        )
-
-    return compilation_result

deploy/docker/routers/tables.py

@@ -1,301 +0,0 @@
-"""
-Table Extraction Router for Crawl4AI Docker Server
-
-This module provides dedicated endpoints for table extraction from HTML or URLs,
-separate from the main crawling functionality.
-"""
-
-import logging
-from typing import List, Dict, Any
-from fastapi import APIRouter, HTTPException
-from fastapi.responses import JSONResponse
-
-# Import crawler pool for browser reuse
-from crawler_pool import get_crawler
-
-# Import schemas
-from schemas import (
-    TableExtractionRequest,
-    TableExtractionBatchRequest,
-    TableExtractionConfig,
-)
-
-# Import utilities
-from utils import (
-    extract_tables_from_html,
-    format_table_response,
-    create_table_extraction_strategy,
-)
-
-# Configure logger
-logger = logging.getLogger(__name__)
-
-# Create router
-router = APIRouter(prefix="/tables", tags=["Table Extraction"])
-
-
-@router.post(
-    "/extract",
-    summary="Extract Tables from HTML or URL",
-    description="""
-Extract tables from HTML content or by fetching a URL.
-Supports multiple extraction strategies: default, LLM-based, or financial.
-    
-**Input Options:**
-- Provide `html` for direct HTML content extraction
-- Provide `url` to fetch and extract from a live page
-- Cannot provide both `html` and `url` simultaneously
-
-**Strategies:**
-- `default`: Fast regex and HTML structure-based extraction
-- `llm`: AI-powered extraction with semantic understanding (requires LLM config)
-- `financial`: Specialized extraction for financial tables with numerical formatting
-
-**Returns:**
-- List of extracted tables with headers, rows, and metadata
-- Each table includes cell-level details and formatting information
-""",
-    response_description="Extracted tables with metadata",
-)
-async def extract_tables(request: TableExtractionRequest) -> JSONResponse:
-    """
-    Extract tables from HTML content or URL.
-
-    Args:
-        request: TableExtractionRequest with html/url and extraction config
-
-    Returns:
-        JSONResponse with extracted tables and metadata
-
-    Raises:
-        HTTPException: If validation fails or extraction errors occur
-    """
-    try:
-        # Validate input
-        if request.html and request.url:
-            raise HTTPException(
-                status_code=400,
-                detail="Cannot provide both 'html' and 'url'. Choose one input method."
-            )
-
-        if not request.html and not request.url:
-            raise HTTPException(
-                status_code=400,
-                detail="Must provide either 'html' or 'url' for table extraction."
-            )
-
-        # Handle URL-based extraction
-        if request.url:
-            # Import crawler configs
-            from async_configs import BrowserConfig, CrawlerRunConfig
-
-            try:
-                # Create minimal browser config
-                browser_config = BrowserConfig(
-                    headless=True,
-                    verbose=False,
-                )
-
-                # Create crawler config with table extraction
-                table_strategy = create_table_extraction_strategy(request.config)
-                crawler_config = CrawlerRunConfig(
-                    table_extraction_strategy=table_strategy,
-                )
-
-                # Get crawler from pool (browser reuse for memory efficiency)
-                crawler = await get_crawler(browser_config, adapter=None)
-                
-                # Crawl the URL
-                result = await crawler.arun(
-                    url=request.url,
-                    config=crawler_config,
-                )
-
-                if not result.success:
-                    raise HTTPException(
-                        status_code=500,
-                        detail=f"Failed to fetch URL: {result.error_message}"
-                    )
-
-                # Extract HTML
-                html_content = result.html
-
-            except Exception as e:
-                logger.error(f"Error fetching URL {request.url}: {e}")
-                raise HTTPException(
-                    status_code=500,
-                    detail=f"Failed to fetch and extract from URL: {str(e)}"
-                )
-
-        else:
-            # Use provided HTML
-            html_content = request.html
-
-        # Extract tables from HTML
-        tables = await extract_tables_from_html(html_content, request.config)
-
-        # Format response
-        formatted_tables = format_table_response(tables)
-
-        return JSONResponse({
-            "success": True,
-            "table_count": len(formatted_tables),
-            "tables": formatted_tables,
-            "strategy": request.config.strategy.value,
-        })
-
-    except HTTPException:
-        raise
-    except Exception as e:
-        logger.error(f"Error extracting tables: {e}", exc_info=True)
-        raise HTTPException(
-            status_code=500,
-            detail=f"Table extraction failed: {str(e)}"
-        )
-
-
-@router.post(
-    "/extract/batch",
-    summary="Extract Tables from Multiple Sources (Batch)",
-    description="""
-Extract tables from multiple HTML contents or URLs in a single request.
-Processes each input independently and returns results for all.
-
-**Batch Processing:**
-- Provide list of HTML contents and/or URLs
-- Each input is processed with the same extraction strategy
-- Partial failures are allowed (returns results for successful extractions)
-
-**Use Cases:**
-- Extracting tables from multiple pages simultaneously
-- Bulk financial data extraction
-- Comparing table structures across multiple sources
-""",
-    response_description="Batch extraction results with per-item success status",
-)
-async def extract_tables_batch(request: TableExtractionBatchRequest) -> JSONResponse:
-    """
-    Extract tables from multiple HTML contents or URLs in batch.
-
-    Args:
-        request: TableExtractionBatchRequest with list of html/url and config
-
-    Returns:
-        JSONResponse with batch results
-
-    Raises:
-        HTTPException: If validation fails
-    """
-    try:
-        # Validate batch request
-        total_items = len(request.html_list or []) + len(request.url_list or [])
-        
-        if total_items == 0:
-            raise HTTPException(
-                status_code=400,
-                detail="Must provide at least one HTML content or URL in batch request."
-            )
-
-        if total_items > 50:  # Reasonable batch limit
-            raise HTTPException(
-                status_code=400,
-                detail=f"Batch size ({total_items}) exceeds maximum allowed (50)."
-            )
-
-        results = []
-
-        # Process HTML list
-        if request.html_list:
-            for idx, html_content in enumerate(request.html_list):
-                try:
-                    tables = await extract_tables_from_html(html_content, request.config)
-                    formatted_tables = format_table_response(tables)
-                    
-                    results.append({
-                        "success": True,
-                        "source": f"html_{idx}",
-                        "table_count": len(formatted_tables),
-                        "tables": formatted_tables,
-                    })
-                except Exception as e:
-                    logger.error(f"Error extracting tables from html_{idx}: {e}")
-                    results.append({
-                        "success": False,
-                        "source": f"html_{idx}",
-                        "error": str(e),
-                    })
-
-        # Process URL list
-        if request.url_list:
-            from async_configs import BrowserConfig, CrawlerRunConfig
-
-            browser_config = BrowserConfig(
-                headless=True,
-                verbose=False,
-            )
-            table_strategy = create_table_extraction_strategy(request.config)
-            crawler_config = CrawlerRunConfig(
-                table_extraction_strategy=table_strategy,
-            )
-
-            # Get crawler from pool (reuse browser for all URLs in batch)
-            crawler = await get_crawler(browser_config, adapter=None)
-            
-            for url in request.url_list:
-                try:
-                    result = await crawler.arun(
-                        url=url,
-                        config=crawler_config,
-                    )
-
-                    if result.success:
-                        html_content = result.html
-                        tables = await extract_tables_from_html(html_content, request.config)
-                        formatted_tables = format_table_response(tables)
-                        
-                        results.append({
-                            "success": True,
-                            "source": url,
-                            "table_count": len(formatted_tables),
-                            "tables": formatted_tables,
-                        })
-                    else:
-                        results.append({
-                            "success": False,
-                            "source": url,
-                            "error": result.error_message,
-                        })
-
-                except Exception as e:
-                    logger.error(f"Error extracting tables from {url}: {e}")
-                    results.append({
-                            "success": False,
-                            "source": url,
-                            "error": str(e),
-                        })
-
-        # Calculate summary
-        successful = sum(1 for r in results if r["success"])
-        failed = len(results) - successful
-        total_tables = sum(r.get("table_count", 0) for r in results if r["success"])
-
-        return JSONResponse({
-            "success": True,
-            "summary": {
-                "total_processed": len(results),
-                "successful": successful,
-                "failed": failed,
-                "total_tables_extracted": total_tables,
-            },
-            "results": results,
-            "strategy": request.config.strategy.value,
-        })
-
-    except HTTPException:
-        raise
-    except Exception as e:
-        logger.error(f"Error in batch table extraction: {e}", exc_info=True)
-        raise HTTPException(
-            status_code=500,
-            detail=f"Batch table extraction failed: {str(e)}"
-        )

deploy/docker/schemas.py

@@ -1,249 +1,28 @@
+from typing import List, Optional, Dict
 from enum import Enum
-from typing import Any, Dict, List, Literal, Optional
-
 from pydantic import BaseModel, Field
 from utils import FilterType
 
 
-# ============================================================================
-# Dispatcher Schemas
-# ============================================================================
-
-class DispatcherType(str, Enum):
-    """Available dispatcher types for crawling."""
-    MEMORY_ADAPTIVE = "memory_adaptive"
-    SEMAPHORE = "semaphore"
-
-
-class DispatcherInfo(BaseModel):
-    """Information about a dispatcher type."""
-    type: DispatcherType
-    name: str
-    description: str
-    config: Dict[str, Any]
-    features: List[str]
-
-
-class DispatcherStatsResponse(BaseModel):
-    """Response model for dispatcher statistics."""
-    type: DispatcherType
-    active_sessions: int
-    config: Dict[str, Any]
-    stats: Optional[Dict[str, Any]] = Field(
-        None, 
-        description="Additional dispatcher-specific statistics"
-    )
-
-
-class DispatcherSelection(BaseModel):
-    """Model for selecting a dispatcher in crawl requests."""
-    dispatcher: Optional[DispatcherType] = Field(
-        None, 
-        description="Dispatcher type to use. Defaults to memory_adaptive if not specified."
-    )
-
-
-# ============================================================================
-# End Dispatcher Schemas
-# ============================================================================
-
-
-# ============================================================================
-# Table Extraction Schemas
-# ============================================================================
-
-class TableExtractionStrategy(str, Enum):
-    """Available table extraction strategies."""
-    NONE = "none"
-    DEFAULT = "default"
-    LLM = "llm"
-    FINANCIAL = "financial"
-
-
-class TableExtractionConfig(BaseModel):
-    """Configuration for table extraction."""
-    
-    strategy: TableExtractionStrategy = Field(
-        default=TableExtractionStrategy.DEFAULT,
-        description="Table extraction strategy to use"
-    )
-    
-    # Common configuration for all strategies
-    table_score_threshold: int = Field(
-        default=7,
-        ge=0,
-        le=100,
-        description="Minimum score for a table to be considered a data table (default strategy)"
-    )
-    min_rows: int = Field(
-        default=0,
-        ge=0,
-        description="Minimum number of rows for a valid table"
-    )
-    min_cols: int = Field(
-        default=0,
-        ge=0,
-        description="Minimum number of columns for a valid table"
-    )
-    
-    # LLM-specific configuration
-    llm_provider: Optional[str] = Field(
-        None,
-        description="LLM provider for LLM strategy (e.g., 'openai/gpt-4')"
-    )
-    llm_model: Optional[str] = Field(
-        None,
-        description="Specific LLM model to use"
-    )
-    llm_api_key: Optional[str] = Field(
-        None,
-        description="API key for LLM provider (if not in environment)"
-    )
-    llm_base_url: Optional[str] = Field(
-        None,
-        description="Custom base URL for LLM API"
-    )
-    extraction_prompt: Optional[str] = Field(
-        None,
-        description="Custom prompt for LLM table extraction"
-    )
-    
-    # Financial-specific configuration
-    decimal_separator: str = Field(
-        default=".",
-        description="Decimal separator for financial tables (e.g., '.' or ',')"
-    )
-    thousand_separator: str = Field(
-        default=",",
-        description="Thousand separator for financial tables (e.g., ',' or '.')"
-    )
-    
-    # General options
-    verbose: bool = Field(
-        default=False,
-        description="Enable verbose logging for table extraction"
-    )
-    
-    class Config:
-        schema_extra = {
-            "example": {
-                "strategy": "default",
-                "table_score_threshold": 7,
-                "min_rows": 2,
-                "min_cols": 2
-            }
-        }
-
-
-class TableExtractionRequest(BaseModel):
-    """Request for dedicated table extraction endpoint."""
-    
-    url: Optional[str] = Field(
-        None,
-        description="URL to crawl and extract tables from"
-    )
-    html: Optional[str] = Field(
-        None,
-        description="Raw HTML content to extract tables from"
-    )
-    config: TableExtractionConfig = Field(
-        default_factory=lambda: TableExtractionConfig(),
-        description="Table extraction configuration"
-    )
-    
-    # Browser config (only used if URL is provided)
-    browser_config: Optional[Dict] = Field(
-        default_factory=dict,
-        description="Browser configuration for URL crawling"
-    )
-    
-    class Config:
-        schema_extra = {
-            "example": {
-                "url": "https://example.com/data-table",
-                "config": {
-                    "strategy": "default",
-                    "min_rows": 2
-                }
-            }
-        }
-
-
-class TableExtractionBatchRequest(BaseModel):
-    """Request for batch table extraction."""
-    
-    html_list: Optional[List[str]] = Field(
-        None,
-        description="List of HTML contents to extract tables from"
-    )
-    url_list: Optional[List[str]] = Field(
-        None,
-        description="List of URLs to extract tables from"
-    )
-    config: TableExtractionConfig = Field(
-        default_factory=lambda: TableExtractionConfig(),
-        description="Table extraction configuration"
-    )
-    browser_config: Optional[Dict] = Field(
-        default_factory=dict,
-        description="Browser configuration"
-    )
-
-
-# ============================================================================
-# End Table Extraction Schemas
-# ============================================================================
-
-
 class CrawlRequest(BaseModel):
     urls: List[str] = Field(min_length=1, max_length=100)
     browser_config: Optional[Dict] = Field(default_factory=dict)
     crawler_config: Optional[Dict] = Field(default_factory=dict)
 
-    anti_bot_strategy: Literal["default", "stealth", "undetected", "max_evasion"] = (
-        Field("default", description="The anti-bot strategy to use for the crawl.")
-    )
-    headless: bool = Field(True, description="Run the browser in headless mode.")
-    
-    # Dispatcher selection
-    dispatcher: Optional[DispatcherType] = Field(
-        None, 
-        description="Dispatcher type to use for crawling. Defaults to memory_adaptive if not specified."
-    )
-    
-    # Proxy rotation configuration
-    proxy_rotation_strategy: Optional[Literal["round_robin", "random", "least_used", "failure_aware"]] = Field(
-        None, description="Proxy rotation strategy to use for the crawl."
-    )
-    proxies: Optional[List[Dict[str, Any]]] = Field(
-        None, description="List of proxy configurations (dicts with server, username, password, etc.)"
-    )
-    proxy_failure_threshold: Optional[int] = Field(
-        3, ge=1, le=10, description="Failure threshold for failure_aware strategy"
-    )
-    proxy_recovery_time: Optional[int] = Field(
-        300, ge=60, le=3600, description="Recovery time in seconds for failure_aware strategy"
-    )
-    
-    # Table extraction configuration
-    table_extraction: Optional[TableExtractionConfig] = Field(
-        None, description="Optional table extraction configuration to extract tables during crawl"
-    )
-
 
 class HookConfig(BaseModel):
     """Configuration for user-provided hooks"""
-
     code: Dict[str, str] = Field(
-        default_factory=dict, description="Map of hook points to Python code strings"
+        default_factory=dict,
+        description="Map of hook points to Python code strings"
     )
     timeout: int = Field(
         default=30,
         ge=1,
         le=120,
-        description="Timeout in seconds for each hook execution",
+        description="Timeout in seconds for each hook execution"
     )
-
+    
     class Config:
         schema_extra = {
             "example": {
@@ -260,81 +39,42 @@ async def hook(page, context, **kwargs):
     await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
     await page.wait_for_timeout(2000)
     return page
-""",
+"""
                 },
-                "timeout": 30,
+                "timeout": 30
             }
         }
 
 
 class CrawlRequestWithHooks(CrawlRequest):
     """Extended crawl request with hooks support"""
-
     hooks: Optional[HookConfig] = Field(
-        default=None, description="Optional user-provided hook functions"
+        default=None,
+        description="Optional user-provided hook functions"
     )
 
-
-class HTTPCrawlRequest(BaseModel):
-    """Request model for HTTP-only crawling endpoints."""
-    
-    urls: List[str] = Field(min_length=1, max_length=100, description="List of URLs to crawl")
-    http_config: Optional[Dict] = Field(
-        default_factory=dict, 
-        description="HTTP crawler configuration (method, headers, timeout, etc.)"
-    )
-    crawler_config: Optional[Dict] = Field(
-        default_factory=dict,
-        description="Crawler run configuration (extraction, filtering, etc.)"
-    )
-    
-    # Dispatcher selection (same as browser crawling)
-    dispatcher: Optional[DispatcherType] = Field(
-        None, 
-        description="Dispatcher type to use. Defaults to memory_adaptive if not specified."
-    )
-
-
-class HTTPCrawlRequestWithHooks(HTTPCrawlRequest):
-    """Extended HTTP crawl request with hooks support"""
-
-    hooks: Optional[HookConfig] = Field(
-        default=None, description="Optional user-provided hook functions"
-    )
-
-
 class MarkdownRequest(BaseModel):
     """Request body for the /md endpoint."""
-
-    url: str = Field(..., description="Absolute http/https URL to fetch")
-    f: FilterType = Field(
-        FilterType.FIT, description="Content‑filter strategy: fit, raw, bm25, or llm"
-    )
-    q: Optional[str] = Field(None, description="Query string used by BM25/LLM filters")
-    c: Optional[str] = Field("0", description="Cache‑bust / revision counter")
-    provider: Optional[str] = Field(
-        None, description="LLM provider override (e.g., 'anthropic/claude-3-opus')"
-    )
-    temperature: Optional[float] = Field(
-        None, description="LLM temperature override (0.0-2.0)"
-    )
+    url: str                    = Field(...,  description="Absolute http/https URL to fetch")
+    f:   FilterType             = Field(FilterType.FIT, description="Content‑filter strategy: fit, raw, bm25, or llm")
+    q:   Optional[str] = Field(None,  description="Query string used by BM25/LLM filters")
+    c:   Optional[str] = Field("0",   description="Cache‑bust / revision counter")
+    provider: Optional[str] = Field(None, description="LLM provider override (e.g., 'anthropic/claude-3-opus')")
+    temperature: Optional[float] = Field(None, description="LLM temperature override (0.0-2.0)")
     base_url: Optional[str] = Field(None, description="LLM API base URL override")
 
 
 class RawCode(BaseModel):
     code: str
 
-
 class HTMLRequest(BaseModel):
     url: str
-
-
+    
 class ScreenshotRequest(BaseModel):
     url: str
     screenshot_wait_for: Optional[float] = 2
     output_path: Optional[str] = None
 
-
 class PDFRequest(BaseModel):
     url: str
     output_path: Optional[str] = None
@@ -343,89 +83,6 @@ class PDFRequest(BaseModel):
 class JSEndpointRequest(BaseModel):
     url: str
     scripts: List[str] = Field(
-        ..., description="List of separated JavaScript snippets to execute"
-    )
-
-
-class SeedRequest(BaseModel):
-    """Request model for URL seeding endpoint."""
-
-    url: str = Field(..., example="https://docs.crawl4ai.com")
-    config: Dict[str, Any] = Field(default_factory=dict)
-
-
-class URLDiscoveryRequest(BaseModel):
-    """Request model for URL discovery endpoint."""
-
-    domain: str = Field(..., example="docs.crawl4ai.com", description="Domain to discover URLs from")
-    seeding_config: Dict[str, Any] = Field(
-        default_factory=dict,
-        description="Configuration for URL discovery using AsyncUrlSeeder",
-        example={
-            "source": "sitemap+cc",
-            "pattern": "*",
-            "live_check": False,
-            "extract_head": False,
-            "max_urls": -1,
-            "concurrency": 1000,
-            "hits_per_sec": 5,
-            "force": False,
-            "verbose": False,
-            "query": None,
-            "score_threshold": None,
-            "scoring_method": "bm25",
-            "filter_nonsense_urls": True
-        }
-    )
-
-
-# --- C4A Script Schemas ---
-
-
-class C4AScriptPayload(BaseModel):
-    """Input model for receiving a C4A-Script."""
-
-    script: str = Field(..., description="The C4A-Script content to process.")
-
-
-# --- Adaptive Crawling Schemas ---
-
-
-class AdaptiveConfigPayload(BaseModel):
-    """Pydantic model for receiving AdaptiveConfig parameters."""
-
-    confidence_threshold: float = 0.7
-    max_pages: int = 20
-    top_k_links: int = 3
-    strategy: str = "statistical"  # "statistical" or "embedding"
-    embedding_model: Optional[str] = "sentence-transformers/all-MiniLM-L6-v2"
-    # Add any other AdaptiveConfig fields you want to expose
-
-
-class AdaptiveCrawlRequest(BaseModel):
-    """Input model for the adaptive digest job."""
-
-    start_url: str = Field(..., description="The starting URL for the adaptive crawl.")
-    query: str = Field(..., description="The user query to guide the crawl.")
-    config: Optional[AdaptiveConfigPayload] = Field(
-        None, description="Optional adaptive crawler configuration."
-    )
-
-
-class AdaptiveJobStatus(BaseModel):
-    """Output model for the job status."""
-
-    task_id: str
-    status: str
-    metrics: Optional[Dict[str, Any]] = None
-    result: Optional[Dict[str, Any]] = None
-    error: Optional[str] = None
-
-
-class LinkAnalysisRequest(BaseModel):
-    """Request body for the /links/analyze endpoint."""
-    url: str = Field(..., description="URL to analyze for links")
-    config: Optional[Dict] = Field(
-        default_factory=dict,
-        description="Optional LinkPreviewConfig dictionary"
-    )
+        ...,
+        description="List of separated JavaScript snippets to execute"
+    )

deploy/docker/utils.py

@@ -6,33 +6,7 @@ from datetime import datetime
 from enum import Enum
 from pathlib import Path
 from fastapi import Request
-from typing import Dict, Optional, Any, List
-
-# Import dispatchers from crawl4ai
-from crawl4ai.async_dispatcher import (
-    BaseDispatcher,
-    MemoryAdaptiveDispatcher,
-    SemaphoreDispatcher,
-)
-
-# Import chunking strategies from crawl4ai
-from crawl4ai.chunking_strategy import (
-    ChunkingStrategy,
-    IdentityChunking,
-    RegexChunking,
-    NlpSentenceChunking,
-    TopicSegmentationChunking,
-    FixedLengthWordChunking,
-    SlidingWindowChunking,
-    OverlappingWindowChunking,
-)
-
-# Import dispatchers from crawl4ai
-from crawl4ai.async_dispatcher import (
-    BaseDispatcher,
-    MemoryAdaptiveDispatcher,
-    SemaphoreDispatcher,
-)
+from typing import Dict, Optional
 
 class TaskStatus(str, Enum):
     PROCESSING = "processing"
@@ -45,124 +19,6 @@ class FilterType(str, Enum):
     BM25 = "bm25"
     LLM = "llm"
 
-
-# ============================================================================
-# Dispatcher Configuration and Factory
-# ============================================================================
-
-# Default dispatcher configurations (hardcoded, no env variables)
-DISPATCHER_DEFAULTS = {
-    "memory_adaptive": {
-        "memory_threshold_percent": 70.0,
-        "critical_threshold_percent": 85.0,
-        "recovery_threshold_percent": 65.0,
-        "check_interval": 1.0,
-        "max_session_permit": 20,
-        "fairness_timeout": 600.0,
-        "memory_wait_timeout": None,  # Disable memory timeout for testing
-    },
-    "semaphore": {
-        "semaphore_count": 5,
-        "max_session_permit": 10,
-    }
-}
-
-DEFAULT_DISPATCHER_TYPE = "memory_adaptive"
-
-
-def create_dispatcher(dispatcher_type: str) -> BaseDispatcher:
-    """
-    Factory function to create dispatcher instances.
-    
-    Args:
-        dispatcher_type: Type of dispatcher to create ("memory_adaptive" or "semaphore")
-        
-    Returns:
-        BaseDispatcher instance
-        
-    Raises:
-        ValueError: If dispatcher type is unknown
-    """
-    dispatcher_type = dispatcher_type.lower()
-    
-    if dispatcher_type == "memory_adaptive":
-        config = DISPATCHER_DEFAULTS["memory_adaptive"]
-        return MemoryAdaptiveDispatcher(
-            memory_threshold_percent=config["memory_threshold_percent"],
-            critical_threshold_percent=config["critical_threshold_percent"],
-            recovery_threshold_percent=config["recovery_threshold_percent"],
-            check_interval=config["check_interval"],
-            max_session_permit=config["max_session_permit"],
-            fairness_timeout=config["fairness_timeout"],
-            memory_wait_timeout=config["memory_wait_timeout"],
-        )
-    elif dispatcher_type == "semaphore":
-        config = DISPATCHER_DEFAULTS["semaphore"]
-        return SemaphoreDispatcher(
-            semaphore_count=config["semaphore_count"],
-            max_session_permit=config["max_session_permit"],
-        )
-    else:
-        raise ValueError(f"Unknown dispatcher type: {dispatcher_type}")
-
-
-def get_dispatcher_config(dispatcher_type: str) -> Dict:
-    """
-    Get configuration for a dispatcher type.
-    
-    Args:
-        dispatcher_type: Type of dispatcher ("memory_adaptive" or "semaphore")
-        
-    Returns:
-        Dictionary containing dispatcher configuration
-        
-    Raises:
-        ValueError: If dispatcher type is unknown
-    """
-    dispatcher_type = dispatcher_type.lower()
-    if dispatcher_type not in DISPATCHER_DEFAULTS:
-        raise ValueError(f"Unknown dispatcher type: {dispatcher_type}")
-    return DISPATCHER_DEFAULTS[dispatcher_type].copy()
-
-
-def get_available_dispatchers() -> Dict[str, Dict]:
-    """
-    Get information about all available dispatchers.
-    
-    Returns:
-        Dictionary mapping dispatcher types to their metadata
-    """
-    return {
-        "memory_adaptive": {
-            "name": "Memory Adaptive Dispatcher",
-            "description": "Dynamically adjusts concurrency based on system memory usage. "
-                          "Monitors memory pressure and adapts crawl sessions accordingly.",
-            "config": DISPATCHER_DEFAULTS["memory_adaptive"],
-            "features": [
-                "Dynamic concurrency adjustment",
-                "Memory pressure monitoring",
-                "Automatic task requeuing under high memory",
-                "Fairness timeout for long-waiting URLs"
-            ]
-        },
-        "semaphore": {
-            "name": "Semaphore Dispatcher",
-            "description": "Fixed concurrency limit using semaphore-based control. "
-                          "Simple and predictable for controlled crawling.",
-            "config": DISPATCHER_DEFAULTS["semaphore"],
-            "features": [
-                "Fixed concurrency limit",
-                "Simple semaphore-based control",
-                "Predictable resource usage"
-            ]
-        }
-    }
-
-# ============================================================================
-# End Dispatcher Configuration
-# ============================================================================
-
-
 def load_config() -> Dict:
     """Load and return application configuration with environment variable overrides."""
     config_path = Path(__file__).parent / "config.yml"
@@ -322,238 +178,4 @@ def verify_email_domain(email: str) -> bool:
         records = dns.resolver.resolve(domain, 'MX')
         return True if records else False
     except Exception as e:
-        return False
-
-
-def create_chunking_strategy(config: Optional[Dict[str, Any]] = None) -> Optional[ChunkingStrategy]:
-    """
-    Factory function to create chunking strategy instances from configuration.
-    
-    Args:
-        config: Dictionary containing 'type' and 'params' keys
-               Example: {"type": "RegexChunking", "params": {"patterns": ["\\n\\n+"]}}
-    
-    Returns:
-        ChunkingStrategy instance or None if config is None
-        
-    Raises:
-        ValueError: If chunking strategy type is unknown or config is invalid
-    """
-    if config is None:
-        return None
-    
-    if not isinstance(config, dict):
-        raise ValueError(f"Chunking strategy config must be a dictionary, got {type(config)}")
-    
-    if "type" not in config:
-        raise ValueError("Chunking strategy config must contain 'type' field")
-    
-    strategy_type = config["type"]
-    params = config.get("params", {})
-    
-    # Validate params is a dict
-    if not isinstance(params, dict):
-        raise ValueError(f"Chunking strategy params must be a dictionary, got {type(params)}")
-    
-    # Strategy factory mapping
-    strategies = {
-        "IdentityChunking": IdentityChunking,
-        "RegexChunking": RegexChunking,
-        "NlpSentenceChunking": NlpSentenceChunking,
-        "TopicSegmentationChunking": TopicSegmentationChunking,
-        "FixedLengthWordChunking": FixedLengthWordChunking,
-        "SlidingWindowChunking": SlidingWindowChunking,
-        "OverlappingWindowChunking": OverlappingWindowChunking,
-    }
-    
-    if strategy_type not in strategies:
-        available = ", ".join(strategies.keys())
-        raise ValueError(f"Unknown chunking strategy type: {strategy_type}. Available: {available}")
-    
-    try:
-        return strategies[strategy_type](**params)
-    except Exception as e:
-        raise ValueError(f"Failed to create {strategy_type} with params {params}: {str(e)}")
-
-
-# ============================================================================
-# Table Extraction Utilities
-# ============================================================================
-
-def create_table_extraction_strategy(config):
-    """
-    Create a table extraction strategy from configuration.
-    
-    Args:
-        config: TableExtractionConfig instance or dict
-        
-    Returns:
-        TableExtractionStrategy instance
-        
-    Raises:
-        ValueError: If strategy type is unknown or configuration is invalid
-    """
-    from crawl4ai.table_extraction import (
-        NoTableExtraction,
-        DefaultTableExtraction,
-        LLMTableExtraction
-    )
-    from schemas import TableExtractionStrategy
-    
-    # Handle both Pydantic model and dict
-    if hasattr(config, 'strategy'):
-        strategy_type = config.strategy
-    elif isinstance(config, dict):
-        strategy_type = config.get('strategy', 'default')
-    else:
-        strategy_type = 'default'
-    
-    # Convert string to enum if needed
-    if isinstance(strategy_type, str):
-        strategy_type = strategy_type.lower()
-    
-    # Extract configuration values
-    def get_config_value(key, default=None):
-        if hasattr(config, key):
-            return getattr(config, key)
-        elif isinstance(config, dict):
-            return config.get(key, default)
-        return default
-    
-    # Create strategy based on type
-    if strategy_type in ['none', TableExtractionStrategy.NONE]:
-        return NoTableExtraction()
-    
-    elif strategy_type in ['default', TableExtractionStrategy.DEFAULT]:
-        return DefaultTableExtraction(
-            table_score_threshold=get_config_value('table_score_threshold', 7),
-            min_rows=get_config_value('min_rows', 0),
-            min_cols=get_config_value('min_cols', 0),
-            verbose=get_config_value('verbose', False)
-        )
-    
-    elif strategy_type in ['llm', TableExtractionStrategy.LLM]:
-        from crawl4ai.types import LLMConfig
-        
-        # Build LLM config
-        llm_config = None
-        llm_provider = get_config_value('llm_provider')
-        llm_api_key = get_config_value('llm_api_key')
-        llm_model = get_config_value('llm_model')
-        llm_base_url = get_config_value('llm_base_url')
-        
-        if llm_provider or llm_api_key:
-            llm_config = LLMConfig(
-                provider=llm_provider or "openai/gpt-4",
-                api_token=llm_api_key,
-                model=llm_model,
-                base_url=llm_base_url
-            )
-        
-        return LLMTableExtraction(
-            llm_config=llm_config,
-            extraction_prompt=get_config_value('extraction_prompt'),
-            table_score_threshold=get_config_value('table_score_threshold', 7),
-            min_rows=get_config_value('min_rows', 0),
-            min_cols=get_config_value('min_cols', 0),
-            verbose=get_config_value('verbose', False)
-        )
-    
-    elif strategy_type in ['financial', TableExtractionStrategy.FINANCIAL]:
-        # Financial strategy uses DefaultTableExtraction with specialized settings
-        # optimized for financial data (tables with currency, numbers, etc.)
-        return DefaultTableExtraction(
-            table_score_threshold=get_config_value('table_score_threshold', 10),  # Higher threshold for financial
-            min_rows=get_config_value('min_rows', 2),  # Financial tables usually have at least 2 rows
-            min_cols=get_config_value('min_cols', 2),  # Financial tables usually have at least 2 columns
-            verbose=get_config_value('verbose', False)
-        )
-    
-    else:
-        raise ValueError(f"Unknown table extraction strategy: {strategy_type}")
-
-
-def format_table_response(tables: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-    """
-    Format extracted tables for API response.
-    
-    Args:
-        tables: List of table dictionaries from table extraction strategy
-        
-    Returns:
-        List of formatted table dictionaries with consistent structure
-    """
-    if not tables:
-        return []
-    
-    formatted_tables = []
-    for idx, table in enumerate(tables):
-        formatted = {
-            "table_index": idx,
-            "headers": table.get("headers", []),
-            "rows": table.get("rows", []),
-            "caption": table.get("caption"),
-            "summary": table.get("summary"),
-            "metadata": table.get("metadata", {}),
-            "row_count": len(table.get("rows", [])),
-            "col_count": len(table.get("headers", [])),
-        }
-        
-        # Add score if available (from scoring strategies)
-        if "score" in table:
-            formatted["score"] = table["score"]
-        
-        # Add position information if available
-        if "position" in table:
-            formatted["position"] = table["position"]
-        
-        formatted_tables.append(formatted)
-    
-    return formatted_tables
-
-
-async def extract_tables_from_html(html: str, config = None):
-    """
-    Extract tables from HTML content (async wrapper for CPU-bound operation).
-    
-    Args:
-        html: HTML content as string
-        config: TableExtractionConfig instance or dict
-        
-    Returns:
-        List of formatted table dictionaries
-        
-    Raises:
-        ValueError: If HTML parsing fails
-    """
-    import asyncio
-    from functools import partial
-    from lxml import html as lxml_html
-    from schemas import TableExtractionConfig
-    
-    # Define sync extraction function
-    def _sync_extract():
-        try:
-            # Parse HTML
-            element = lxml_html.fromstring(html)
-        except Exception as e:
-            raise ValueError(f"Failed to parse HTML: {str(e)}")
-        
-        # Create strategy
-        cfg = config if config is not None else TableExtractionConfig()
-        strategy = create_table_extraction_strategy(cfg)
-        
-        # Extract tables
-        tables = strategy.extract_tables(element)
-        
-        # Format response
-        return format_table_response(tables)
-    
-    # Run in executor to avoid blocking the event loop
-    loop = asyncio.get_event_loop()
-    return await loop.run_in_executor(None, _sync_extract)
-
-
-# ============================================================================
-# End Table Extraction Utilities
-# ============================================================================
+        return False

docs/PROXY_ROTATION_STRATEGY_DOCS.md

@@ -1,431 +0,0 @@
-# Proxy Rotation Strategy Documentation
-
-## Overview
-
-The Crawl4AI FastAPI server now includes comprehensive proxy rotation functionality that allows you to distribute requests across multiple proxy servers using different rotation strategies. This feature helps prevent IP blocking, distributes load across proxy infrastructure, and provides redundancy for high-availability crawling operations.
-
-## Available Proxy Rotation Strategies
-
-| Strategy | Description | Use Case | Performance |
-|----------|-------------|----------|-------------|
-| `round_robin` | Cycles through proxies sequentially | Even distribution, predictable pattern | ⭐⭐⭐⭐⭐ |
-| `random` | Randomly selects from available proxies | Unpredictable traffic pattern | ⭐⭐⭐⭐ |
-| `least_used` | Uses proxy with lowest usage count | Optimal load balancing | ⭐⭐⭐ |
-| `failure_aware` | Avoids failed proxies with auto-recovery | High availability, fault tolerance | ⭐⭐⭐⭐ |
-
-## API Endpoints
-
-### POST /crawl
-
-Standard crawling endpoint with proxy rotation support.
-
-**Request Body:**
-```json
-{
-  "urls": ["https://example.com"],
-  "proxy_rotation_strategy": "round_robin",
-  "proxies": [
-    {"server": "http://proxy1.com:8080", "username": "user1", "password": "pass1"},
-    {"server": "http://proxy2.com:8080", "username": "user2", "password": "pass2"}
-  ],
-  "browser_config": {},
-  "crawler_config": {}
-}
-```
-
-### POST /crawl/stream
-
-Streaming crawling endpoint with proxy rotation support.
-
-**Request Body:**
-```json
-{
-  "urls": ["https://example.com"],
-  "proxy_rotation_strategy": "failure_aware",
-  "proxy_failure_threshold": 3,
-  "proxy_recovery_time": 300,
-  "proxies": [
-    {"server": "http://proxy1.com:8080", "username": "user1", "password": "pass1"},
-    {"server": "http://proxy2.com:8080", "username": "user2", "password": "pass2"}
-  ],
-  "browser_config": {},
-  "crawler_config": {
-    "stream": true
-  }
-}
-```
-
-## Parameters
-
-### proxy_rotation_strategy (optional)
-- **Type:** `string`
-- **Default:** `null` (no proxy rotation)
-- **Options:** `"round_robin"`, `"random"`, `"least_used"`, `"failure_aware"`
-- **Description:** Selects the proxy rotation strategy for distributing requests
-
-### proxies (optional)
-- **Type:** `array of objects`
-- **Default:** `null`
-- **Description:** List of proxy configurations to rotate between
-- **Required when:** `proxy_rotation_strategy` is specified
-
-### proxy_failure_threshold (optional)
-- **Type:** `integer`
-- **Default:** `3`
-- **Range:** `1-10`
-- **Description:** Number of failures before marking a proxy as unhealthy (failure_aware only)
-
-### proxy_recovery_time (optional)
-- **Type:** `integer`
-- **Default:** `300` (5 minutes)
-- **Range:** `60-3600` seconds
-- **Description:** Time to wait before attempting to use a failed proxy again (failure_aware only)
-
-## Proxy Configuration Format
-
-### Full Configuration
-```json
-{
-  "server": "http://proxy.example.com:8080",
-  "username": "proxy_user",
-  "password": "proxy_pass",
-  "ip": "192.168.1.100"
-}
-```
-
-### Minimal Configuration
-```json
-{
-  "server": "http://192.168.1.100:8080"
-}
-```
-
-### SOCKS Proxy Support
-```json
-{
-  "server": "socks5://127.0.0.1:1080",
-  "username": "socks_user",
-  "password": "socks_pass"
-}
-```
-
-## Usage Examples
-
-### 1. Round Robin Strategy
-
-```bash
-curl -X POST "http://localhost:11235/crawl" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://httpbin.org/ip"],
-    "proxy_rotation_strategy": "round_robin",
-    "proxies": [
-      {"server": "http://proxy1.com:8080", "username": "user1", "password": "pass1"},
-      {"server": "http://proxy2.com:8080", "username": "user2", "password": "pass2"},
-      {"server": "http://proxy3.com:8080", "username": "user3", "password": "pass3"}
-    ]
-  }'
-```
-
-### 2. Random Strategy with Minimal Config
-
-```bash
-curl -X POST "http://localhost:11235/crawl" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://httpbin.org/headers"],
-    "proxy_rotation_strategy": "random",
-    "proxies": [
-      {"server": "http://192.168.1.100:8080"},
-      {"server": "http://192.168.1.101:8080"},
-      {"server": "http://192.168.1.102:8080"}
-    ]
-  }'
-```
-
-### 3. Least Used Strategy with Load Balancing
-
-```bash
-curl -X POST "http://localhost:11235/crawl" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://example.com", "https://httpbin.org/html", "https://httpbin.org/json"],
-    "proxy_rotation_strategy": "least_used",
-    "proxies": [
-      {"server": "http://proxy1.com:8080", "username": "user1", "password": "pass1"},
-      {"server": "http://proxy2.com:8080", "username": "user2", "password": "pass2"}
-    ],
-    "crawler_config": {
-      "cache_mode": "bypass"
-    }
-  }'
-```
-
-### 4. Failure-Aware Strategy with High Availability
-
-```bash
-curl -X POST "http://localhost:11235/crawl" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://example.com"],
-    "proxy_rotation_strategy": "failure_aware",
-    "proxy_failure_threshold": 2,
-    "proxy_recovery_time": 180,
-    "proxies": [
-      {"server": "http://proxy1.com:8080", "username": "user1", "password": "pass1"},
-      {"server": "http://proxy2.com:8080", "username": "user2", "password": "pass2"},
-      {"server": "http://proxy3.com:8080", "username": "user3", "password": "pass3"}
-    ],
-    "headless": true
-  }'
-```
-
-### 5. Streaming with Proxy Rotation
-
-```bash
-curl -X POST "http://localhost:11235/crawl/stream" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://example.com", "https://httpbin.org/html"],
-    "proxy_rotation_strategy": "round_robin",
-    "proxies": [
-      {"server": "http://proxy1.com:8080", "username": "user1", "password": "pass1"},
-      {"server": "http://proxy2.com:8080", "username": "user2", "password": "pass2"}
-    ],
-    "crawler_config": {
-      "stream": true,
-      "cache_mode": "bypass"
-    }
-  }'
-```
-
-## Combining with Anti-Bot Strategies
-
-You can combine proxy rotation with anti-bot strategies for maximum effectiveness:
-
-```bash
-curl -X POST "http://localhost:11235/crawl" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://protected-site.com"],
-    "anti_bot_strategy": "stealth",
-    "proxy_rotation_strategy": "failure_aware",
-    "proxy_failure_threshold": 2,
-    "proxies": [
-      {"server": "http://proxy1.com:8080", "username": "user1", "password": "pass1"},
-      {"server": "http://proxy2.com:8080", "username": "user2", "password": "pass2"}
-    ],
-    "headless": true,
-    "browser_config": {
-      "enable_stealth": true
-    }
-  }'
-```
-
-## Strategy Details
-
-### Round Robin Strategy
-- **Algorithm:** Sequential cycling through proxy list
-- **Pros:** Predictable, even distribution, simple
-- **Cons:** Predictable pattern may be detectable
-- **Best for:** General use, development, testing
-
-### Random Strategy
-- **Algorithm:** Random selection from available proxies
-- **Pros:** Unpredictable pattern, good for evasion
-- **Cons:** Uneven distribution possible
-- **Best for:** Anti-detection, varying traffic patterns
-
-### Least Used Strategy
-- **Algorithm:** Selects proxy with minimum usage count
-- **Pros:** Optimal load balancing, prevents overloading
-- **Cons:** Slightly more complex, tracking overhead
-- **Best for:** High-volume crawling, load balancing
-
-### Failure-Aware Strategy
-- **Algorithm:** Tracks proxy health, auto-recovery
-- **Pros:** High availability, fault tolerance, automatic recovery
-- **Cons:** Most complex, memory overhead for tracking
-- **Best for:** Production environments, critical crawling
-
-## Error Handling
-
-### Common Errors
-
-#### Invalid Proxy Configuration
-```json
-{
-  "error": "Invalid proxy configuration: Proxy configuration missing 'server' field: {'username': 'user1'}"
-}
-```
-
-#### Unsupported Strategy
-```json
-{
-  "error": "Unsupported proxy rotation strategy: invalid_strategy. Available: round_robin, random, least_used, failure_aware"
-}
-```
-
-#### Missing Proxies
-When `proxy_rotation_strategy` is specified but `proxies` is empty:
-```json
-{
-  "error": "proxy_rotation_strategy specified but no proxies provided"
-}
-```
-
-## Environment Variable Support
-
-You can also configure proxies using environment variables:
-
-```bash
-# Set proxy list (comma-separated)
-export PROXIES="proxy1.com:8080:user1:pass1,proxy2.com:8080:user2:pass2"
-
-# Set default strategy
-export PROXY_ROTATION_STRATEGY="round_robin"
-```
-
-## Performance Considerations
-
-1. **Strategy Overhead:**
-   - Round Robin: Minimal overhead
-   - Random: Low overhead
-   - Least Used: Medium overhead (usage tracking)
-   - Failure Aware: High overhead (health tracking)
-
-2. **Memory Usage:**
-   - Round Robin: ~O(n) where n = number of proxies
-   - Random: ~O(n)
-   - Least Used: ~O(n) + usage counters
-   - Failure Aware: ~O(n) + health tracking data
-
-3. **Concurrent Safety:**
-   - All strategies are async-safe with proper locking
-   - No race conditions in proxy selection
-
-## Best Practices
-
-1. **Production Deployment:**
-   - Use `failure_aware` strategy for high availability
-   - Set appropriate failure thresholds (2-3)
-   - Use recovery times between 3-10 minutes
-
-2. **Development/Testing:**
-   - Use `round_robin` for predictable behavior
-   - Start with small proxy pools (2-3 proxies)
-
-3. **Anti-Detection:**
-   - Combine with `stealth` or `undetected` anti-bot strategies
-   - Use `random` strategy for unpredictable patterns
-   - Vary proxy geographic locations
-
-4. **Load Balancing:**
-   - Use `least_used` for even distribution
-   - Monitor proxy performance and adjust pools accordingly
-
-5. **Error Monitoring:**
-   - Monitor failure rates with `failure_aware` strategy
-   - Set up alerts for proxy pool depletion
-   - Implement fallback mechanisms
-
-## Integration Examples
-
-### Python Requests
-```python
-import requests
-
-payload = {
-    "urls": ["https://example.com"],
-    "proxy_rotation_strategy": "round_robin",
-    "proxies": [
-        {"server": "http://proxy1.com:8080", "username": "user1", "password": "pass1"},
-        {"server": "http://proxy2.com:8080", "username": "user2", "password": "pass2"}
-    ]
-}
-
-response = requests.post("http://localhost:11235/crawl", json=payload)
-print(response.json())
-```
-
-### JavaScript/Node.js
-```javascript
-const axios = require('axios');
-
-const payload = {
-  urls: ["https://example.com"],
-  proxy_rotation_strategy: "failure_aware",
-  proxy_failure_threshold: 2,
-  proxies: [
-    {server: "http://proxy1.com:8080", username: "user1", password: "pass1"},
-    {server: "http://proxy2.com:8080", username: "user2", password: "pass2"}
-  ]
-};
-
-axios.post('http://localhost:11235/crawl', payload)
-  .then(response => console.log(response.data))
-  .catch(error => console.error(error));
-```
-
-### cURL with Multiple URLs
-```bash
-curl -X POST "http://localhost:11235/crawl" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": [
-      "https://example.com",
-      "https://httpbin.org/html",
-      "https://httpbin.org/json",
-      "https://httpbin.org/xml"
-    ],
-    "proxy_rotation_strategy": "least_used",
-    "proxies": [
-      {"server": "http://proxy1.com:8080", "username": "user1", "password": "pass1"},
-      {"server": "http://proxy2.com:8080", "username": "user2", "password": "pass2"},
-      {"server": "http://proxy3.com:8080", "username": "user3", "password": "pass3"}
-    ],
-    "crawler_config": {
-      "cache_mode": "bypass",
-      "wait_for_images": false
-    }
-  }'
-```
-
-## Troubleshooting
-
-### Common Issues
-
-1. **All proxies failing:**
-   - Check proxy connectivity
-   - Verify authentication credentials
-   - Ensure proxy servers support the target protocols
-
-2. **Uneven distribution:**
-   - Use `least_used` strategy for better balancing
-   - Monitor proxy usage patterns
-
-3. **High memory usage:**
-   - Reduce proxy pool size
-   - Consider using `round_robin` instead of `failure_aware`
-
-4. **Slow performance:**
-   - Check proxy response times
-   - Use geographically closer proxies
-   - Reduce failure thresholds
-
-### Debug Information
-
-Enable verbose logging to see proxy selection details:
-
-```json
-{
-  "urls": ["https://example.com"],
-  "proxy_rotation_strategy": "failure_aware",
-  "proxies": [...],
-  "crawler_config": {
-    "verbose": true
-  }
-}
-```
-
-This will log which proxy is selected for each request and any failure/recovery events.

docs/examples/docker_client_hooks_example.py

@@ -0,0 +1,522 @@
+#!/usr/bin/env python3
+"""
+Comprehensive hooks examples using Docker Client with function objects.
+
+This approach is recommended because:
+- Write hooks as regular Python functions
+- Full IDE support (autocomplete, type checking)
+- Automatic conversion to API format
+- Reusable and testable code
+- Clean, readable syntax
+"""
+
+import asyncio
+from crawl4ai import Crawl4aiDockerClient
+
+# API_BASE_URL = "http://localhost:11235"
+API_BASE_URL = "http://localhost:11234"
+
+
+# ============================================================================
+# Hook Function Definitions
+# ============================================================================
+
+# --- All Hooks Demo ---
+async def browser_created_hook(browser, **kwargs):
+    """Called after browser is created"""
+    print("[HOOK] Browser created and ready")
+    return browser
+
+
+async def page_context_hook(page, context, **kwargs):
+    """Setup page environment"""
+    print("[HOOK] Setting up page environment")
+
+    # Set viewport
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+
+    # Add cookies
+    await context.add_cookies([{
+        "name": "test_session",
+        "value": "abc123xyz",
+        "domain": ".httpbin.org",
+        "path": "/"
+    }])
+
+    # Block resources
+    await context.route("**/*.{png,jpg,jpeg,gif}", lambda route: route.abort())
+    await context.route("**/analytics/*", lambda route: route.abort())
+
+    print("[HOOK] Environment configured")
+    return page
+
+
+async def user_agent_hook(page, context, user_agent, **kwargs):
+    """Called when user agent is updated"""
+    print(f"[HOOK] User agent: {user_agent[:50]}...")
+    return page
+
+
+async def before_goto_hook(page, context, url, **kwargs):
+    """Called before navigating to URL"""
+    print(f"[HOOK] Navigating to: {url}")
+
+    await page.set_extra_http_headers({
+        "X-Custom-Header": "crawl4ai-test",
+        "Accept-Language": "en-US"
+    })
+
+    return page
+
+
+async def after_goto_hook(page, context, url, response, **kwargs):
+    """Called after page loads"""
+    print(f"[HOOK] Page loaded: {url}")
+
+    await page.wait_for_timeout(1000)
+
+    try:
+        await page.wait_for_selector("body", timeout=2000)
+        print("[HOOK] Body element ready")
+    except:
+        print("[HOOK] Timeout, continuing")
+
+    return page
+
+
+async def execution_started_hook(page, context, **kwargs):
+    """Called when custom JS execution starts"""
+    print("[HOOK] JS execution started")
+    await page.evaluate("console.log('[HOOK] Custom JS');")
+    return page
+
+
+async def before_retrieve_hook(page, context, **kwargs):
+    """Called before retrieving HTML"""
+    print("[HOOK] Preparing HTML retrieval")
+
+    # Scroll for lazy content
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
+    await page.wait_for_timeout(500)
+    await page.evaluate("window.scrollTo(0, 0);")
+
+    print("[HOOK] Scrolling complete")
+    return page
+
+
+async def before_return_hook(page, context, html, **kwargs):
+    """Called before returning HTML"""
+    print(f"[HOOK] HTML ready: {len(html)} chars")
+
+    metrics = await page.evaluate('''() => ({
+        images: document.images.length,
+        links: document.links.length,
+        scripts: document.scripts.length
+    })''')
+
+    print(f"[HOOK] Metrics - Images: {metrics['images']}, Links: {metrics['links']}")
+    return page
+
+
+# --- Authentication Hooks ---
+async def auth_context_hook(page, context, **kwargs):
+    """Setup authentication context"""
+    print("[HOOK] Setting up authentication")
+
+    # Add auth cookies
+    await context.add_cookies([{
+        "name": "auth_token",
+        "value": "fake_jwt_token",
+        "domain": ".httpbin.org",
+        "path": "/",
+        "httpOnly": True
+    }])
+
+    # Set localStorage
+    await page.evaluate('''
+        localStorage.setItem('user_id', '12345');
+        localStorage.setItem('auth_time', new Date().toISOString());
+    ''')
+
+    print("[HOOK] Auth context ready")
+    return page
+
+
+async def auth_headers_hook(page, context, url, **kwargs):
+    """Add authentication headers"""
+    print(f"[HOOK] Adding auth headers for {url}")
+
+    import base64
+    credentials = base64.b64encode(b"user:passwd").decode('ascii')
+
+    await page.set_extra_http_headers({
+        'Authorization': f'Basic {credentials}',
+        'X-API-Key': 'test-key-123'
+    })
+
+    return page
+
+
+# --- Performance Optimization Hooks ---
+async def performance_hook(page, context, **kwargs):
+    """Optimize page for performance"""
+    print("[HOOK] Optimizing for performance")
+
+    # Block resource-heavy content
+    await context.route("**/*.{png,jpg,jpeg,gif,webp,svg}", lambda r: r.abort())
+    await context.route("**/*.{woff,woff2,ttf}", lambda r: r.abort())
+    await context.route("**/*.{mp4,webm,ogg}", lambda r: r.abort())
+    await context.route("**/googletagmanager.com/*", lambda r: r.abort())
+    await context.route("**/google-analytics.com/*", lambda r: r.abort())
+    await context.route("**/facebook.com/*", lambda r: r.abort())
+
+    # Disable animations
+    await page.add_style_tag(content='''
+        *, *::before, *::after {
+            animation-duration: 0s !important;
+            transition-duration: 0s !important;
+        }
+    ''')
+
+    print("[HOOK] Optimizations applied")
+    return page
+
+
+async def cleanup_hook(page, context, **kwargs):
+    """Clean page before extraction"""
+    print("[HOOK] Cleaning page")
+
+    await page.evaluate('''() => {
+        const selectors = [
+            '.ad', '.ads', '.advertisement',
+            '.popup', '.modal', '.overlay',
+            '.cookie-banner', '.newsletter'
+        ];
+
+        selectors.forEach(sel => {
+            document.querySelectorAll(sel).forEach(el => el.remove());
+        });
+
+        document.querySelectorAll('script, style').forEach(el => el.remove());
+    }''')
+
+    print("[HOOK] Page cleaned")
+    return page
+
+
+# --- Content Extraction Hooks ---
+async def wait_dynamic_content_hook(page, context, url, response, **kwargs):
+    """Wait for dynamic content to load"""
+    print(f"[HOOK] Waiting for dynamic content on {url}")
+
+    await page.wait_for_timeout(2000)
+
+    # Click "Load More" if exists
+    try:
+        load_more = await page.query_selector('[class*="load-more"], button:has-text("Load More")')
+        if load_more:
+            await load_more.click()
+            await page.wait_for_timeout(1000)
+            print("[HOOK] Clicked 'Load More'")
+    except:
+        pass
+
+    return page
+
+
+async def extract_metadata_hook(page, context, **kwargs):
+    """Extract page metadata"""
+    print("[HOOK] Extracting metadata")
+
+    metadata = await page.evaluate('''() => {
+        const getMeta = (name) => {
+            const el = document.querySelector(`meta[name="${name}"], meta[property="${name}"]`);
+            return el ? el.getAttribute('content') : null;
+        };
+
+        return {
+            title: document.title,
+            description: getMeta('description'),
+            author: getMeta('author'),
+            keywords: getMeta('keywords'),
+        };
+    }''')
+
+    print(f"[HOOK] Metadata: {metadata}")
+
+    # Infinite scroll
+    for i in range(3):
+        await page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
+        await page.wait_for_timeout(1000)
+        print(f"[HOOK] Scroll {i+1}/3")
+
+    return page
+
+
+# --- Multi-URL Hooks ---
+async def url_specific_hook(page, context, url, **kwargs):
+    """Apply URL-specific logic"""
+    print(f"[HOOK] Processing URL: {url}")
+
+    # URL-specific headers
+    if 'html' in url:
+        await page.set_extra_http_headers({"X-Type": "HTML"})
+    elif 'json' in url:
+        await page.set_extra_http_headers({"X-Type": "JSON"})
+
+    return page
+
+
+async def track_progress_hook(page, context, url, response, **kwargs):
+    """Track crawl progress"""
+    status = response.status if response else 'unknown'
+    print(f"[HOOK] Loaded {url} - Status: {status}")
+    return page
+
+
+# ============================================================================
+# Test Functions
+# ============================================================================
+
+async def test_all_hooks_comprehensive():
+    """Test all 8 hook types"""
+    print("=" * 70)
+    print("Test 1: All Hooks Comprehensive Demo (Docker Client)")
+    print("=" * 70)
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nCrawling with all 8 hooks...")
+
+        # Define hooks with function objects
+        hooks = {
+            "on_browser_created": browser_created_hook,
+            "on_page_context_created": page_context_hook,
+            "on_user_agent_updated": user_agent_hook,
+            "before_goto": before_goto_hook,
+            "after_goto": after_goto_hook,
+            "on_execution_started": execution_started_hook,
+            "before_retrieve_html": before_retrieve_hook,
+            "before_return_html": before_return_hook
+        }
+
+        result = await client.crawl(
+            ["https://httpbin.org/html"],
+            hooks=hooks,
+            hooks_timeout=30
+        )
+
+        print("\n✅ Success!")
+        print(f"   URL: {result.url}")
+        print(f"   Success: {result.success}")
+        print(f"   HTML: {len(result.html)} chars")
+
+
+async def test_authentication_workflow():
+    """Test authentication with hooks"""
+    print("\n" + "=" * 70)
+    print("Test 2: Authentication Workflow (Docker Client)")
+    print("=" * 70)
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nTesting authentication...")
+
+        hooks = {
+            "on_page_context_created": auth_context_hook,
+            "before_goto": auth_headers_hook
+        }
+
+        result = await client.crawl(
+            ["https://httpbin.org/basic-auth/user/passwd"],
+            hooks=hooks,
+            hooks_timeout=15
+        )
+
+        print("\n✅ Authentication completed")
+
+        if result.success:
+            if '"authenticated"' in result.html and 'true' in result.html:
+                print("   ✅ Basic auth successful!")
+            else:
+                print("   ⚠️ Auth status unclear")
+        else:
+            print(f"   ❌ Failed: {result.error_message}")
+
+
+async def test_performance_optimization():
+    """Test performance optimization"""
+    print("\n" + "=" * 70)
+    print("Test 3: Performance Optimization (Docker Client)")
+    print("=" * 70)
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nTesting performance hooks...")
+
+        hooks = {
+            "on_page_context_created": performance_hook,
+            "before_retrieve_html": cleanup_hook
+        }
+
+        result = await client.crawl(
+            ["https://httpbin.org/html"],
+            hooks=hooks,
+            hooks_timeout=10
+        )
+
+        print("\n✅ Optimization completed")
+        print(f"   HTML size: {len(result.html):,} chars")
+        print("   Resources blocked, ads removed")
+
+
+async def test_content_extraction():
+    """Test content extraction"""
+    print("\n" + "=" * 70)
+    print("Test 4: Content Extraction (Docker Client)")
+    print("=" * 70)
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nTesting extraction hooks...")
+
+        hooks = {
+            "after_goto": wait_dynamic_content_hook,
+            "before_retrieve_html": extract_metadata_hook
+        }
+
+        result = await client.crawl(
+            ["https://www.kidocode.com/"],
+            hooks=hooks,
+            hooks_timeout=20
+        )
+
+        print("\n✅ Extraction completed")
+        print(f"   URL: {result.url}")
+        print(f"   Success: {result.success}")
+        print(f"   Metadata: {result.metadata}")
+
+
+async def test_multi_url_crawl():
+    """Test hooks with multiple URLs"""
+    print("\n" + "=" * 70)
+    print("Test 5: Multi-URL Crawl (Docker Client)")
+    print("=" * 70)
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nCrawling multiple URLs...")
+
+        hooks = {
+            "before_goto": url_specific_hook,
+            "after_goto": track_progress_hook
+        }
+
+        results = await client.crawl(
+            [
+                "https://httpbin.org/html",
+                "https://httpbin.org/json",
+                "https://httpbin.org/xml"
+            ],
+            hooks=hooks,
+            hooks_timeout=15
+        )
+
+        print("\n✅ Multi-URL crawl completed")
+        print(f"\n   Crawled {len(results)} URLs:")
+        for i, result in enumerate(results, 1):
+            status = "✅" if result.success else "❌"
+            print(f"   {status} {i}. {result.url}")
+
+
+async def test_reusable_hook_library():
+    """Test using reusable hook library"""
+    print("\n" + "=" * 70)
+    print("Test 6: Reusable Hook Library (Docker Client)")
+    print("=" * 70)
+
+    # Create a library of reusable hooks
+    class HookLibrary:
+        @staticmethod
+        async def block_images(page, context, **kwargs):
+            """Block all images"""
+            await context.route("**/*.{png,jpg,jpeg,gif}", lambda r: r.abort())
+            print("[LIBRARY] Images blocked")
+            return page
+
+        @staticmethod
+        async def block_analytics(page, context, **kwargs):
+            """Block analytics"""
+            await context.route("**/analytics/*", lambda r: r.abort())
+            await context.route("**/google-analytics.com/*", lambda r: r.abort())
+            print("[LIBRARY] Analytics blocked")
+            return page
+
+        @staticmethod
+        async def scroll_infinite(page, context, **kwargs):
+            """Handle infinite scroll"""
+            for i in range(5):
+                prev = await page.evaluate("document.body.scrollHeight")
+                await page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
+                await page.wait_for_timeout(1000)
+                curr = await page.evaluate("document.body.scrollHeight")
+                if curr == prev:
+                    break
+            print("[LIBRARY] Infinite scroll complete")
+            return page
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nUsing hook library...")
+
+        hooks = {
+            "on_page_context_created": HookLibrary.block_images,
+            "before_retrieve_html": HookLibrary.scroll_infinite
+        }
+
+        result = await client.crawl(
+            ["https://www.kidocode.com/"],
+            hooks=hooks,
+            hooks_timeout=20
+        )
+
+        print("\n✅ Library hooks completed")
+        print(f"   Success: {result.success}")
+
+
+# ============================================================================
+# Main
+# ============================================================================
+
+async def main():
+    """Run all Docker client hook examples"""
+    print("🔧 Crawl4AI Docker Client - Hooks Examples (Function-Based)")
+    print("Using Python function objects with automatic conversion")
+    print("=" * 70)
+
+    tests = [
+        ("All Hooks Demo", test_all_hooks_comprehensive),
+        ("Authentication", test_authentication_workflow),
+        ("Performance", test_performance_optimization),
+        ("Extraction", test_content_extraction),
+        ("Multi-URL", test_multi_url_crawl),
+        ("Hook Library", test_reusable_hook_library)
+    ]
+
+    for i, (name, test_func) in enumerate(tests, 1):
+        try:
+            await test_func()
+            print(f"\n✅ Test {i}/{len(tests)}: {name} completed\n")
+        except Exception as e:
+            print(f"\n❌ Test {i}/{len(tests)}: {name} failed: {e}\n")
+            import traceback
+            traceback.print_exc()
+
+    print("=" * 70)
+    print("🎉 All Docker client hook examples completed!")
+    print("\n💡 Key Benefits of Function-Based Hooks:")
+    print("   • Write as regular Python functions")
+    print("   • Full IDE support (autocomplete, types)")
+    print("   • Automatic conversion to API format")
+    print("   • Reusable across projects")
+    print("   • Clean, readable code")
+    print("   • Easy to test and debug")
+    print("=" * 70)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/link_analysis_example.py

@@ -1,315 +0,0 @@
-#!/usr/bin/env python3
-"""
-Link Analysis Example
-====================
-
-This example demonstrates how to use the new /links/analyze endpoint
-to extract, analyze, and score links from web pages.
-
-Requirements:
-- Crawl4AI server running on localhost:11234
-- requests library: pip install requests
-"""
-
-import requests
-import json
-import time
-from typing import Dict, Any, List
-
-
-class LinkAnalyzer:
-    """Simple client for the link analysis endpoint"""
-
-    def __init__(self, base_url: str = "http://localhost:11234", token: str = None):
-        self.base_url = base_url
-        self.token = token or self._get_test_token()
-
-    def _get_test_token(self) -> str:
-        """Get a test token (for development only)"""
-        try:
-            response = requests.post(
-                f"{self.base_url}/token",
-                json={"email": "test@example.com"},
-                timeout=10
-            )
-            if response.status_code == 200:
-                return response.json()["access_token"]
-        except:
-            pass
-        return "test-token"  # Fallback for local testing
-
-    def analyze_links(self, url: str, config: Dict[str, Any] = None) -> Dict[str, Any]:
-        """Analyze links on a webpage"""
-        headers = {"Content-Type": "application/json"}
-
-        if self.token and self.token != "test-token":
-            headers["Authorization"] = f"Bearer {self.token}"
-
-        data = {"url": url}
-        if config:
-            data["config"] = config
-
-        response = requests.post(
-            f"{self.base_url}/links/analyze",
-            headers=headers,
-            json=data,
-            timeout=30
-        )
-
-        response.raise_for_status()
-        return response.json()
-
-    def print_summary(self, result: Dict[str, Any]):
-        """Print a summary of link analysis results"""
-        print("\n" + "="*60)
-        print("📊 LINK ANALYSIS SUMMARY")
-        print("="*60)
-
-        total_links = sum(len(links) for links in result.values())
-        print(f"Total links found: {total_links}")
-
-        for category, links in result.items():
-            if links:
-                print(f"\n📂 {category.upper()}: {len(links)} links")
-
-                # Show top 3 links by score
-                top_links = sorted(links, key=lambda x: x.get('total_score', 0), reverse=True)[:3]
-                for i, link in enumerate(top_links, 1):
-                    score = link.get('total_score', 0)
-                    text = link.get('text', 'No text')[:50]
-                    url = link.get('href', 'No URL')[:60]
-                    print(f"  {i}. [{score:.2f}] {text} → {url}")
-
-
-def example_1_basic_analysis():
-    """Example 1: Basic link analysis"""
-    print("\n🔍 Example 1: Basic Link Analysis")
-    print("-" * 40)
-
-    analyzer = LinkAnalyzer()
-
-    # Analyze a simple test page
-    url = "https://httpbin.org/links/10"
-    print(f"Analyzing: {url}")
-
-    try:
-        result = analyzer.analyze_links(url)
-        analyzer.print_summary(result)
-        return result
-    except Exception as e:
-        print(f"❌ Error: {e}")
-        return None
-
-
-def example_2_custom_config():
-    """Example 2: Analysis with custom configuration"""
-    print("\n🔍 Example 2: Custom Configuration")
-    print("-" * 40)
-
-    analyzer = LinkAnalyzer()
-
-    # Custom configuration
-    config = {
-        "include_internal": True,
-        "include_external": True,
-        "max_links": 50,
-        "timeout": 10,
-        "verbose": True
-    }
-
-    url = "https://httpbin.org/links/10"
-    print(f"Analyzing with custom config: {url}")
-    print(f"Config: {json.dumps(config, indent=2)}")
-
-    try:
-        result = analyzer.analyze_links(url, config)
-        analyzer.print_summary(result)
-        return result
-    except Exception as e:
-        print(f"❌ Error: {e}")
-        return None
-
-
-def example_3_real_world_site():
-    """Example 3: Analyzing a real website"""
-    print("\n🔍 Example 3: Real Website Analysis")
-    print("-" * 40)
-
-    analyzer = LinkAnalyzer()
-
-    # Analyze Python official website
-    url = "https://www.python.org"
-    print(f"Analyzing real website: {url}")
-    print("This may take a moment...")
-
-    try:
-        result = analyzer.analyze_links(url)
-        analyzer.print_summary(result)
-
-        # Additional analysis
-        print("\n📈 DETAILED ANALYSIS")
-        print("-" * 20)
-
-        # Find external links with highest scores
-        external_links = result.get('external', [])
-        if external_links:
-            top_external = sorted(external_links, key=lambda x: x.get('total_score', 0), reverse=True)[:5]
-            print("\n🌐 Top External Links:")
-            for link in top_external:
-                print(f"  • {link.get('text', 'N/A')} (score: {link.get('total_score', 0):.2f})")
-                print(f"    {link.get('href', 'N/A')}")
-
-        # Find internal links
-        internal_links = result.get('internal', [])
-        if internal_links:
-            top_internal = sorted(internal_links, key=lambda x: x.get('total_score', 0), reverse=True)[:5]
-            print("\n🏠 Top Internal Links:")
-            for link in top_internal:
-                print(f"  • {link.get('text', 'N/A')} (score: {link.get('total_score', 0):.2f})")
-                print(f"    {link.get('href', 'N/A')}")
-
-        return result
-    except Exception as e:
-        print(f"❌ Error: {e}")
-        print("⚠️  This example may fail due to network issues")
-        return None
-
-
-def example_4_comparative_analysis():
-    """Example 4: Comparing link structures across sites"""
-    print("\n🔍 Example 4: Comparative Analysis")
-    print("-" * 40)
-
-    analyzer = LinkAnalyzer()
-
-    sites = [
-        ("https://httpbin.org/links/10", "Test Page 1"),
-        ("https://httpbin.org/links/5", "Test Page 2")
-    ]
-
-    results = {}
-
-    for url, name in sites:
-        print(f"\nAnalyzing: {name}")
-        try:
-            result = analyzer.analyze_links(url)
-            results[name] = result
-
-            total_links = sum(len(links) for links in result.values())
-            categories = len([cat for cat, links in result.items() if links])
-            print(f"  Links: {total_links}, Categories: {categories}")
-
-        except Exception as e:
-            print(f"  ❌ Error: {e}")
-
-    # Compare results
-    if len(results) > 1:
-        print("\n📊 COMPARISON")
-        print("-" * 15)
-
-        for name, result in results.items():
-            total = sum(len(links) for links in result.values())
-            print(f"{name}: {total} total links")
-
-            # Calculate average scores
-            all_scores = []
-            for links in result.values():
-                for link in links:
-                    all_scores.append(link.get('total_score', 0))
-
-            if all_scores:
-                avg_score = sum(all_scores) / len(all_scores)
-                print(f"  Average link score: {avg_score:.3f}")
-
-
-def example_5_advanced_filtering():
-    """Example 5: Advanced filtering and analysis"""
-    print("\n🔍 Example 5: Advanced Filtering")
-    print("-" * 40)
-
-    analyzer = LinkAnalyzer()
-
-    url = "https://httpbin.org/links/10"
-
-    try:
-        result = analyzer.analyze_links(url)
-
-        # Filter links by score
-        min_score = 0.5
-        high_quality_links = {}
-
-        for category, links in result.items():
-            if links:
-                filtered = [link for link in links if link.get('total_score', 0) >= min_score]
-                if filtered:
-                    high_quality_links[category] = filtered
-
-        print(f"\n🎯 High-quality links (score >= {min_score}):")
-        total_high_quality = sum(len(links) for links in high_quality_links.values())
-        print(f"Total: {total_high_quality} links")
-
-        for category, links in high_quality_links.items():
-            print(f"\n{category.upper()}:")
-            for link in links:
-                score = link.get('total_score', 0)
-                text = link.get('text', 'No text')
-                print(f"  • [{score:.2f}] {text}")
-
-        # Extract unique domains from external links
-        external_links = result.get('external', [])
-        if external_links:
-            domains = set()
-            for link in external_links:
-                url = link.get('href', '')
-                if '://' in url:
-                    domain = url.split('://')[1].split('/')[0]
-                    domains.add(domain)
-
-            print(f"\n🌐 Unique external domains: {len(domains)}")
-            for domain in sorted(domains):
-                print(f"  • {domain}")
-
-    except Exception as e:
-        print(f"❌ Error: {e}")
-
-
-def main():
-    """Run all examples"""
-    print("🚀 Link Analysis Examples")
-    print("=" * 50)
-    print("Make sure the Crawl4AI server is running on localhost:11234")
-    print()
-
-    examples = [
-        example_1_basic_analysis,
-        example_2_custom_config,
-        example_3_real_world_site,
-        example_4_comparative_analysis,
-        example_5_advanced_filtering
-    ]
-
-    for i, example_func in enumerate(examples, 1):
-        print(f"\n{'='*60}")
-        print(f"Running Example {i}")
-        print('='*60)
-
-        try:
-            example_func()
-        except KeyboardInterrupt:
-            print("\n⏹️  Example interrupted by user")
-            break
-        except Exception as e:
-            print(f"\n❌ Example {i} failed: {e}")
-
-        if i < len(examples):
-            print("\n⏳ Press Enter to continue to next example...")
-            try:
-                input()
-            except KeyboardInterrupt:
-                break
-
-    print("\n🎉 Examples completed!")
-
-
-if __name__ == "__main__":
-    main()

docs/examples/table-extraction-api.md

@@ -1,626 +0,0 @@
-# Table Extraction API Documentation
-
-## Overview
-
-The Crawl4AI Docker Server provides powerful table extraction capabilities through both **integrated** and **dedicated** endpoints. Extract structured data from HTML tables using multiple strategies: default (fast regex-based), LLM-powered (semantic understanding), or financial (specialized for financial data).
-
----
-
-## Table of Contents
-
-1. [Quick Start](#quick-start)
-2. [Extraction Strategies](#extraction-strategies)
-3. [Integrated Extraction (with /crawl)](#integrated-extraction)
-4. [Dedicated Endpoints (/tables)](#dedicated-endpoints)
-5. [Batch Processing](#batch-processing)
-6. [Configuration Options](#configuration-options)
-7. [Response Format](#response-format)
-8. [Error Handling](#error-handling)
-
----
-
-## Quick Start
-
-### Extract Tables During Crawl
-
-```bash
-curl -X POST http://localhost:11235/crawl \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://example.com/financial-data"],
-    "table_extraction": {
-      "strategy": "default"
-    }
-  }'
-```
-
-### Extract Tables from HTML
-
-```bash
-curl -X POST http://localhost:11235/tables/extract \
-  -H "Content-Type: application/json" \
-  -d '{
-    "html": "<table><tr><th>Name</th><th>Value</th></tr><tr><td>A</td><td>100</td></tr></table>",
-    "config": {
-      "strategy": "default"
-    }
-  }'
-```
-
----
-
-## Extraction Strategies
-
-### 1. **Default Strategy** (Fast, Regex-Based)
-
-Best for general-purpose table extraction with high performance.
-
-```json
-{
-  "strategy": "default"
-}
-```
-
-**Use Cases:**
-- General web scraping
-- Simple data tables
-- High-volume extraction
-
-### 2. **LLM Strategy** (AI-Powered)
-
-Uses Large Language Models for semantic understanding and complex table structures.
-
-```json
-{
-  "strategy": "llm",
-  "llm_provider": "openai",
-  "llm_model": "gpt-4",
-  "llm_api_key": "your-api-key",
-  "llm_prompt": "Extract and structure the financial data"
-}
-```
-
-**Use Cases:**
-- Complex nested tables
-- Tables with irregular structure
-- Semantic data extraction
-
-**Supported Providers:**
-- `openai` (GPT-3.5, GPT-4)
-- `anthropic` (Claude)
-- `huggingface` (Open models)
-
-### 3. **Financial Strategy** (Specialized)
-
-Optimized for financial tables with proper numerical formatting.
-
-```json
-{
-  "strategy": "financial",
-  "preserve_formatting": true,
-  "extract_metadata": true
-}
-```
-
-**Use Cases:**
-- Stock data
-- Financial statements
-- Accounting tables
-- Price lists
-
-### 4. **None Strategy** (No Extraction)
-
-Disables table extraction.
-
-```json
-{
-  "strategy": "none"
-}
-```
-
----
-
-## Integrated Extraction
-
-Add table extraction to any crawl request by including the `table_extraction` configuration.
-
-### Example: Basic Integration
-
-```python
-import requests
-
-response = requests.post("http://localhost:11235/crawl", json={
-    "urls": ["https://finance.yahoo.com/quote/AAPL"],
-    "browser_config": {
-        "headless": True
-    },
-    "crawler_config": {
-        "wait_until": "networkidle"
-    },
-    "table_extraction": {
-        "strategy": "financial",
-        "preserve_formatting": True
-    }
-})
-
-data = response.json()
-for result in data["results"]:
-    if result["success"]:
-        print(f"Found {len(result.get('tables', []))} tables")
-        for table in result.get("tables", []):
-            print(f"Table: {table['headers']}")
-```
-
-### Example: Multiple URLs with Table Extraction
-
-```javascript
-// Node.js example
-const axios = require('axios');
-
-const response = await axios.post('http://localhost:11235/crawl', {
-  urls: [
-    'https://example.com/page1',
-    'https://example.com/page2',
-    'https://example.com/page3'
-  ],
-  table_extraction: {
-    strategy: 'default'
-  }
-});
-
-response.data.results.forEach((result, index) => {
-  console.log(`Page ${index + 1}:`);
-  console.log(`  Tables found: ${result.tables?.length || 0}`);
-});
-```
-
-### Example: LLM-Based Extraction with Custom Prompt
-
-```bash
-curl -X POST http://localhost:11235/crawl \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://example.com/complex-data"],
-    "table_extraction": {
-      "strategy": "llm",
-      "llm_provider": "openai",
-      "llm_model": "gpt-4",
-      "llm_api_key": "sk-...",
-      "llm_prompt": "Extract product pricing information, including discounts and availability"
-    }
-  }'
-```
-
----
-
-## Dedicated Endpoints
-
-### `/tables/extract` - Single Extraction
-
-Extract tables from HTML content or by fetching a URL.
-
-#### Extract from HTML
-
-```python
-import requests
-
-html_content = """
-<table>
-  <thead>
-    <tr><th>Product</th><th>Price</th><th>Stock</th></tr>
-  </thead>
-  <tbody>
-    <tr><td>Widget A</td><td>$19.99</td><td>In Stock</td></tr>
-    <tr><td>Widget B</td><td>$29.99</td><td>Out of Stock</td></tr>
-  </tbody>
-</table>
-"""
-
-response = requests.post("http://localhost:11235/tables/extract", json={
-    "html": html_content,
-    "config": {
-        "strategy": "default"
-    }
-})
-
-data = response.json()
-print(f"Success: {data['success']}")
-print(f"Tables found: {data['table_count']}")
-print(f"Strategy used: {data['strategy']}")
-
-for table in data['tables']:
-    print("\nTable:")
-    print(f"  Headers: {table['headers']}")
-    print(f"  Rows: {len(table['rows'])}")
-```
-
-#### Extract from URL
-
-```python
-response = requests.post("http://localhost:11235/tables/extract", json={
-    "url": "https://example.com/data-page",
-    "config": {
-        "strategy": "financial",
-        "preserve_formatting": True
-    }
-})
-
-data = response.json()
-for table in data['tables']:
-    print(f"Table with {len(table['rows'])} rows")
-```
-
----
-
-## Batch Processing
-
-### `/tables/extract/batch` - Batch Extraction
-
-Extract tables from multiple HTML contents or URLs in a single request.
-
-#### Batch from HTML List
-
-```python
-import requests
-
-html_contents = [
-    "<table><tr><th>A</th></tr><tr><td>1</td></tr></table>",
-    "<table><tr><th>B</th></tr><tr><td>2</td></tr></table>",
-    "<table><tr><th>C</th></tr><tr><td>3</td></tr></table>",
-]
-
-response = requests.post("http://localhost:11235/tables/extract/batch", json={
-    "html_list": html_contents,
-    "config": {
-        "strategy": "default"
-    }
-})
-
-data = response.json()
-print(f"Total processed: {data['summary']['total_processed']}")
-print(f"Successful: {data['summary']['successful']}")
-print(f"Failed: {data['summary']['failed']}")
-print(f"Total tables: {data['summary']['total_tables_extracted']}")
-
-for result in data['results']:
-    if result['success']:
-        print(f"  {result['source']}: {result['table_count']} tables")
-    else:
-        print(f"  {result['source']}: Error - {result['error']}")
-```
-
-#### Batch from URL List
-
-```python
-response = requests.post("http://localhost:11235/tables/extract/batch", json={
-    "url_list": [
-        "https://example.com/page1",
-        "https://example.com/page2",
-        "https://example.com/page3",
-    ],
-    "config": {
-        "strategy": "financial"
-    }
-})
-
-data = response.json()
-for result in data['results']:
-    print(f"URL: {result['source']}")
-    if result['success']:
-        print(f"  ✓ Found {result['table_count']} tables")
-    else:
-        print(f"  ✗ Failed: {result['error']}")
-```
-
-#### Mixed Batch (HTML + URLs)
-
-```python
-response = requests.post("http://localhost:11235/tables/extract/batch", json={
-    "html_list": [
-        "<table><tr><th>Local</th></tr></table>"
-    ],
-    "url_list": [
-        "https://example.com/remote"
-    ],
-    "config": {
-        "strategy": "default"
-    }
-})
-```
-
-**Batch Limits:**
-- Maximum 50 items per batch request
-- Items are processed independently (partial failures allowed)
-
----
-
-## Configuration Options
-
-### TableExtractionConfig
-
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `strategy` | `"none"` \| `"default"` \| `"llm"` \| `"financial"` | `"default"` | Extraction strategy to use |
-| `llm_provider` | `string` | `null` | LLM provider (required for `llm` strategy) |
-| `llm_model` | `string` | `null` | Model name (required for `llm` strategy) |
-| `llm_api_key` | `string` | `null` | API key (required for `llm` strategy) |
-| `llm_prompt` | `string` | `null` | Custom extraction prompt |
-| `preserve_formatting` | `boolean` | `false` | Keep original number/date formatting |
-| `extract_metadata` | `boolean` | `false` | Include table metadata (id, class, etc.) |
-
-### Example: Full Configuration
-
-```json
-{
-  "strategy": "llm",
-  "llm_provider": "openai",
-  "llm_model": "gpt-4",
-  "llm_api_key": "sk-...",
-  "llm_prompt": "Extract structured product data",
-  "preserve_formatting": true,
-  "extract_metadata": true
-}
-```
-
----
-
-## Response Format
-
-### Single Extraction Response
-
-```json
-{
-  "success": true,
-  "table_count": 2,
-  "strategy": "default",
-  "tables": [
-    {
-      "headers": ["Product", "Price", "Stock"],
-      "rows": [
-        ["Widget A", "$19.99", "In Stock"],
-        ["Widget B", "$29.99", "Out of Stock"]
-      ],
-      "metadata": {
-        "id": "product-table",
-        "class": "data-table",
-        "row_count": 2,
-        "column_count": 3
-      }
-    }
-  ]
-}
-```
-
-### Batch Extraction Response
-
-```json
-{
-  "success": true,
-  "summary": {
-    "total_processed": 3,
-    "successful": 2,
-    "failed": 1,
-    "total_tables_extracted": 5
-  },
-  "strategy": "default",
-  "results": [
-    {
-      "success": true,
-      "source": "html_0",
-      "table_count": 2,
-      "tables": [...]
-    },
-    {
-      "success": true,
-      "source": "https://example.com",
-      "table_count": 3,
-      "tables": [...]
-    },
-    {
-      "success": false,
-      "source": "html_2",
-      "error": "Invalid HTML structure"
-    }
-  ]
-}
-```
-
-### Integrated Crawl Response
-
-Tables are included in the standard crawl result:
-
-```json
-{
-  "success": true,
-  "results": [
-    {
-      "url": "https://example.com",
-      "success": true,
-      "html": "...",
-      "markdown": "...",
-      "tables": [
-        {
-          "headers": [...],
-          "rows": [...]
-        }
-      ]
-    }
-  ]
-}
-```
-
----
-
-## Error Handling
-
-### Common Errors
-
-#### 400 Bad Request
-
-```json
-{
-  "detail": "Must provide either 'html' or 'url' for table extraction."
-}
-```
-
-**Cause:** Invalid request parameters
-
-**Solution:** Ensure you provide exactly one of `html` or `url`
-
-#### 400 Bad Request (LLM)
-
-```json
-{
-  "detail": "Invalid table extraction config: LLM strategy requires llm_provider, llm_model, and llm_api_key"
-}
-```
-
-**Cause:** Missing required LLM configuration
-
-**Solution:** Provide all required LLM fields
-
-#### 500 Internal Server Error
-
-```json
-{
-  "detail": "Failed to fetch and extract from URL: Connection timeout"
-}
-```
-
-**Cause:** URL fetch failure or extraction error
-
-**Solution:** Check URL accessibility and HTML validity
-
-### Handling Partial Failures in Batch
-
-```python
-response = requests.post("http://localhost:11235/tables/extract/batch", json={
-    "url_list": urls,
-    "config": {"strategy": "default"}
-})
-
-data = response.json()
-
-successful_results = [r for r in data['results'] if r['success']]
-failed_results = [r for r in data['results'] if not r['success']]
-
-print(f"Successful: {len(successful_results)}")
-for result in failed_results:
-    print(f"Failed: {result['source']} - {result['error']}")
-```
-
----
-
-## Best Practices
-
-### 1. **Choose the Right Strategy**
-
-- **Default**: Fast, reliable for most tables
-- **LLM**: Complex structures, semantic extraction
-- **Financial**: Numerical data with formatting
-
-### 2. **Batch Processing**
-
-- Use batch endpoints for multiple pages
-- Keep batch size under 50 items
-- Handle partial failures gracefully
-
-### 3. **Performance Optimization**
-
-- Use `default` strategy for high-volume extraction
-- Enable `preserve_formatting` only when needed
-- Limit `extract_metadata` to reduce payload size
-
-### 4. **LLM Strategy Tips**
-
-- Use specific prompts for better results
-- GPT-4 for complex tables, GPT-3.5 for simple ones
-- Cache results to reduce API costs
-
-### 5. **Error Handling**
-
-- Always check `success` field
-- Log errors for debugging
-- Implement retry logic for transient failures
-
----
-
-## Examples by Use Case
-
-### Financial Data Extraction
-
-```python
-response = requests.post("http://localhost:11235/crawl", json={
-    "urls": ["https://finance.site.com/stocks"],
-    "table_extraction": {
-        "strategy": "financial",
-        "preserve_formatting": True,
-        "extract_metadata": True
-    }
-})
-
-for result in response.json()["results"]:
-    for table in result.get("tables", []):
-        # Financial tables with preserved formatting
-        print(table["rows"])
-```
-
-### Product Catalog Scraping
-
-```python
-response = requests.post("http://localhost:11235/tables/extract/batch", json={
-    "url_list": [
-        "https://shop.com/category/electronics",
-        "https://shop.com/category/clothing",
-        "https://shop.com/category/books",
-    ],
-    "config": {"strategy": "default"}
-})
-
-all_products = []
-for result in response.json()["results"]:
-    if result["success"]:
-        for table in result["tables"]:
-            all_products.extend(table["rows"])
-
-print(f"Total products: {len(all_products)}")
-```
-
-### Complex Table with LLM
-
-```python
-response = requests.post("http://localhost:11235/tables/extract", json={
-    "url": "https://complex-data.com/report",
-    "config": {
-        "strategy": "llm",
-        "llm_provider": "openai",
-        "llm_model": "gpt-4",
-        "llm_api_key": "sk-...",
-        "llm_prompt": "Extract quarterly revenue breakdown by region and product category"
-    }
-})
-
-structured_data = response.json()["tables"]
-```
-
----
-
-## API Reference Summary
-
-| Endpoint | Method | Purpose |
-|----------|--------|---------|
-| `/crawl` | POST | Crawl with integrated table extraction |
-| `/crawl/stream` | POST | Stream crawl with table extraction |
-| `/tables/extract` | POST | Extract tables from HTML or URL |
-| `/tables/extract/batch` | POST | Batch extract from multiple sources |
-
-For complete API documentation, visit: `/docs` (Swagger UI)
-
----
-
-## Support
-
-For issues, feature requests, or questions:
-- GitHub: https://github.com/unclecode/crawl4ai
-- Documentation: https://crawl4ai.com/docs
-- Discord: https://discord.gg/crawl4ai

docs/md_v2/assets/page_actions.css

@@ -0,0 +1,376 @@
+/* ==== File: assets/page_actions.css ==== */
+/* Page Actions Dropdown - Terminal Style */
+
+/* Wrapper - positioned in content area */
+.page-actions-wrapper {
+    position: absolute;
+    top: 1.3rem;
+    right: 1rem;
+    z-index: 1000;
+}
+
+/* Floating Action Button */
+.page-actions-button {
+    position: relative;
+    display: inline-flex;
+    align-items: center;
+    gap: 0.5rem;
+    background: #3f3f44;
+    border: 1px solid #50ffff;
+    color: #e8e9ed;
+    padding: 0.75rem 1rem;
+    border-radius: 6px;
+    font-family: 'Dank Mono', Monaco, monospace;
+    font-size: 0.875rem;
+    cursor: pointer;
+    transition: all 0.2s ease;
+    box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3);
+}
+
+.page-actions-button:hover {
+    background: #50ffff;
+    color: #070708;
+    transform: translateY(-2px);
+    box-shadow: 0 6px 16px rgba(80, 255, 255, 0.3);
+}
+
+.page-actions-button::before {
+    content: '▤';
+    font-size: 1.2rem;
+    line-height: 1;
+}
+
+.page-actions-button::after {
+    content: '▼';
+    font-size: 0.6rem;
+    transition: transform 0.2s ease;
+}
+
+.page-actions-button.active::after {
+    transform: rotate(180deg);
+}
+
+/* Dropdown Menu */
+.page-actions-dropdown {
+    position: absolute;
+    top: 3.5rem;
+    right: 0;
+    z-index: 1001;
+    background: #1a1a1a;
+    border: 1px solid #3f3f44;
+    border-radius: 8px;
+    min-width: 280px;
+    opacity: 0;
+    visibility: hidden;
+    transform: translateY(-10px);
+    transition: all 0.2s ease;
+    box-shadow: 0 8px 24px rgba(0, 0, 0, 0.5);
+    overflow: hidden;
+}
+
+.page-actions-dropdown.active {
+    opacity: 1;
+    visibility: visible;
+    transform: translateY(0);
+}
+
+.page-actions-dropdown::before {
+    content: '';
+    position: absolute;
+    top: -8px;
+    right: 1.5rem;
+    width: 0;
+    height: 0;
+    border-left: 8px solid transparent;
+    border-right: 8px solid transparent;
+    border-bottom: 8px solid #3f3f44;
+}
+
+/* Menu Header */
+.page-actions-header {
+    background: #3f3f44;
+    padding: 0.5rem 0.75rem;
+    border-bottom: 1px solid #50ffff;
+    font-family: 'Dank Mono', Monaco, monospace;
+    font-size: 0.7rem;
+    color: #a3abba;
+    text-transform: uppercase;
+    letter-spacing: 0.05em;
+}
+
+.page-actions-header::before {
+    content: '┌─';
+    margin-right: 0.5rem;
+    color: #50ffff;
+}
+
+/* Menu Items */
+.page-actions-menu {
+    list-style: none;
+    margin: 0;
+    padding: 0.25rem 0;
+}
+
+.page-action-item {
+    display: block;
+    padding: 0;
+}
+
+ul>li.page-action-item::after{
+    content: '';
+}
+.page-action-link {
+    display: flex;
+    align-items: center;
+    gap: 0.5rem;
+    padding: 0.5rem 0.75rem;
+    color: #e8e9ed;
+    text-decoration: none !important;
+    font-family: 'Dank Mono', Monaco, monospace;
+    font-size: 0.8rem;
+    transition: all 0.15s ease;
+    cursor: pointer;
+    border-left: 3px solid transparent;
+}
+
+.page-action-link:hover:not(.disabled) {
+    background: #3f3f44;
+    border-left-color: #50ffff;
+    color: #50ffff;
+    text-decoration: none;
+}
+
+.page-action-link.disabled {
+    opacity: 0.5;
+    cursor: not-allowed;
+}
+
+.page-action-link.disabled:hover {
+    background: transparent;
+    color: #e8e9ed;
+    text-decoration: none;
+}
+
+/* Icons using ASCII/Terminal characters */
+.page-action-icon {
+    font-size: 1rem;
+    width: 1.5rem;
+    text-align: center;
+    font-weight: bold;
+    color: #50ffff;
+}
+
+.page-action-link:hover:not(.disabled) .page-action-icon {
+    color: #50ffff;
+}
+
+.page-action-link.disabled .page-action-icon {
+    color: #666;
+}
+
+/* Specific icons */
+.icon-copy::before {
+    content: '⎘'; /* Copy/duplicate symbol */
+}
+
+.icon-view::before {
+    content: '⎙'; /* Document symbol */
+}
+
+.icon-ai::before {
+    content: '⚡'; /* Lightning/AI symbol */
+}
+
+/* Action Text */
+.page-action-text {
+    flex: 1;
+}
+
+.page-action-label {
+    display: block;
+    font-weight: 600;
+    margin-bottom: 0.05rem;
+    line-height: 1.3;
+}
+
+.page-action-description {
+    display: block;
+    font-size: 0.7rem;
+    color: #a3abba;
+    line-height: 1.2;
+}
+
+/* Badge */
+/* External link indicator */
+.page-action-external::after {
+    content: '→';
+    margin-left: 0.25rem;
+    font-size: 0.75rem;
+}
+
+/* Divider */
+.page-actions-divider {
+    height: 1px;
+    background: #3f3f44;
+    margin: 0.25rem 0;
+}
+
+/* Success/Copy feedback */
+.page-action-copied {
+    background: #50ff50 !important;
+    color: #070708 !important;
+    border-left-color: #50ff50 !important;
+}
+
+.page-action-copied .page-action-icon {
+    color: #070708 !important;
+}
+
+.page-action-copied .page-action-icon::before {
+    content: '✓';
+}
+
+/* Mobile Responsive */
+@media (max-width: 768px) {
+    .page-actions-wrapper {
+        top: 0.5rem;
+        right: 0.5rem;
+    }
+
+    .page-actions-button {
+        padding: 0.6rem 0.8rem;
+        font-size: 0.8rem;
+    }
+
+    .page-actions-dropdown {
+        min-width: 260px;
+        max-width: calc(100vw - 2rem);
+        right: -0.5rem;
+    }
+
+    .page-action-link {
+        padding: 0.6rem 0.8rem;
+        font-size: 0.8rem;
+    }
+
+    .page-action-description {
+        font-size: 0.7rem;
+    }
+}
+
+/* Animation for tooltip/notification */
+@keyframes slideInFromTop {
+    from {
+        transform: translateY(-20px);
+        opacity: 0;
+    }
+    to {
+        transform: translateY(0);
+        opacity: 1;
+    }
+}
+
+.page-actions-notification {
+    position: fixed;
+    top: calc(var(--header-height) + 0.5rem);
+    right: 50%;
+    transform: translateX(50%);
+    z-index: 1100;
+    background: #50ff50;
+    color: #070708;
+    padding: 0.75rem 1.5rem;
+    border-radius: 6px;
+    font-family: 'Dank Mono', Monaco, monospace;
+    font-size: 0.875rem;
+    font-weight: 600;
+    box-shadow: 0 4px 12px rgba(80, 255, 80, 0.4);
+    animation: slideInFromTop 0.3s ease;
+    pointer-events: none;
+}
+
+.page-actions-notification::before {
+    content: '✓ ';
+    margin-right: 0.5rem;
+}
+
+/* Hide on print */
+@media print {
+    .page-actions-button,
+    .page-actions-dropdown {
+        display: none !important;
+    }
+}
+
+/* Overlay for mobile */
+.page-actions-overlay {
+    display: none;
+    position: fixed;
+    top: 0;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    background: rgba(0, 0, 0, 0.5);
+    z-index: 998;
+    opacity: 0;
+    transition: opacity 0.2s ease;
+}
+
+.page-actions-overlay.active {
+    display: block;
+    opacity: 1;
+}
+
+@media (max-width: 768px) {
+    .page-actions-overlay {
+        display: block;
+    }
+}
+
+/* Keyboard focus styles */
+.page-action-link:focus {
+    outline: 2px solid #50ffff;
+    outline-offset: -2px;
+}
+
+.page-actions-button:focus {
+    outline: 2px solid #50ffff;
+    outline-offset: 2px;
+}
+
+/* Loading state */
+.page-action-link.loading {
+    pointer-events: none;
+    opacity: 0.7;
+}
+
+.page-action-link.loading .page-action-icon::before {
+    content: '⟳';
+    animation: spin 1s linear infinite;
+}
+
+@keyframes spin {
+    from { transform: rotate(0deg); }
+    to { transform: rotate(360deg); }
+}
+
+/* Terminal-style border effect on hover */
+.page-actions-dropdown:hover {
+    border-color: #50ffff;
+}
+
+/* Footer info */
+.page-actions-footer {
+    background: #070708;
+    padding: 0.4rem 0.75rem;
+    border-top: 1px solid #3f3f44;
+    font-size: 0.65rem;
+    color: #666;
+    text-align: center;
+    font-family: 'Dank Mono', Monaco, monospace;
+}
+
+.page-actions-footer::before {
+    content: '└─';
+    margin-right: 0.5rem;
+    color: #3f3f44;
+}

docs/md_v2/assets/page_actions.js

@@ -0,0 +1,427 @@
+// ==== File: assets/page_actions.js ====
+// Page Actions - Copy/View Markdown functionality
+
+document.addEventListener('DOMContentLoaded', () => {
+    // Configuration
+    const config = {
+        githubRepo: 'unclecode/crawl4ai',
+        githubBranch: 'main',
+        docsPath: 'docs/md_v2',
+        excludePaths: ['/apps/c4a-script/', '/apps/llmtxt/', '/apps/crawl4ai-assistant/', '/core/ask-ai/'], // Don't show on app pages
+    };
+
+    let cachedMarkdown = null;
+    let cachedMarkdownPath = null;
+
+    // Check if we should show the button on this page
+    function shouldShowButton() {
+        const currentPath = window.location.pathname;
+
+        // Don't show on homepage
+        if (currentPath === '/' || currentPath === '/index.html') {
+            return false;
+        }
+
+        // Don't show on 404 pages
+        if (document.title && document.title.toLowerCase().includes('404')) {
+            return false;
+        }
+
+        // Require mkdocs main content container
+        const mainContent = document.getElementById('terminal-mkdocs-main-content');
+        if (!mainContent) {
+            return false;
+        }
+
+        // Don't show on excluded paths (apps)
+        for (const excludePath of config.excludePaths) {
+            if (currentPath.includes(excludePath)) {
+                return false;
+            }
+        }
+
+        // Only show on documentation pages
+        return true;
+    }
+
+    if (!shouldShowButton()) {
+        return;
+    }
+
+    // Get current page markdown path
+    function getCurrentMarkdownPath() {
+        let path = window.location.pathname;
+
+        // Remove leading/trailing slashes
+        path = path.replace(/^\/|\/$/g, '');
+
+        // Remove .html extension if present
+        path = path.replace(/\.html$/, '');
+
+        // Handle root/index
+        if (!path || path === 'index') {
+            return 'index.md';
+        }
+
+        // Add .md extension
+        return `${path}.md`;
+    }
+
+    async function loadMarkdownContent() {
+        const mdPath = getCurrentMarkdownPath();
+
+        if (!mdPath) {
+            throw new Error('Invalid markdown path');
+        }
+
+        const rawUrl = getGithubRawUrl();
+        const response = await fetch(rawUrl);
+
+        if (!response.ok) {
+            throw new Error(`Failed to fetch markdown: ${response.status}`);
+        }
+
+        const markdown = await response.text();
+        cachedMarkdown = markdown;
+        cachedMarkdownPath = mdPath;
+        return markdown;
+    }
+
+    async function ensureMarkdownCached() {
+        const mdPath = getCurrentMarkdownPath();
+
+        if (!mdPath) {
+            return false;
+        }
+
+        if (cachedMarkdown && cachedMarkdownPath === mdPath) {
+            return true;
+        }
+
+        try {
+            await loadMarkdownContent();
+            return true;
+        } catch (error) {
+            console.warn('Page Actions: Markdown not available for this page.', error);
+            cachedMarkdown = null;
+            cachedMarkdownPath = null;
+            return false;
+        }
+    }
+
+    async function getMarkdownContent() {
+        const available = await ensureMarkdownCached();
+        if (!available) {
+            throw new Error('Markdown not available for this page.');
+        }
+        return cachedMarkdown;
+    }
+
+    // Get GitHub raw URL for current page
+    function getGithubRawUrl() {
+        const mdPath = getCurrentMarkdownPath();
+        return `https://raw.githubusercontent.com/${config.githubRepo}/${config.githubBranch}/${config.docsPath}/${mdPath}`;
+    }
+
+    // Get GitHub file URL for current page (for viewing)
+    function getGithubFileUrl() {
+        const mdPath = getCurrentMarkdownPath();
+        return `https://github.com/${config.githubRepo}/blob/${config.githubBranch}/${config.docsPath}/${mdPath}`;
+    }
+
+    // Create the UI
+    function createPageActionsUI() {
+        // Find the main content area
+        const mainContent = document.getElementById('terminal-mkdocs-main-content');
+        if (!mainContent) {
+            console.warn('Page Actions: Could not find #terminal-mkdocs-main-content');
+            return null;
+        }
+
+        // Create button
+        const button = document.createElement('button');
+        button.className = 'page-actions-button';
+        button.setAttribute('aria-label', 'Page copy');
+        button.setAttribute('aria-expanded', 'false');
+        button.innerHTML = '<span>Page Copy</span>';
+
+        // Create overlay for mobile
+        const overlay = document.createElement('div');
+        overlay.className = 'page-actions-overlay';
+
+        // Create dropdown
+        const dropdown = document.createElement('div');
+        dropdown.className = 'page-actions-dropdown';
+        dropdown.setAttribute('role', 'menu');
+        dropdown.innerHTML = `
+            <div class="page-actions-header">Page Copy</div>
+            <ul class="page-actions-menu">
+                <li class="page-action-item">
+                    <a href="#" class="page-action-link" id="action-copy-markdown" role="menuitem">
+                        <span class="page-action-icon icon-copy"></span>
+                        <span class="page-action-text">
+                            <span class="page-action-label">Copy as Markdown</span>
+                            <span class="page-action-description">Copy page for LLMs</span>
+                        </span>
+                    </a>
+                </li>
+                <li class="page-action-item">
+                    <a href="#" class="page-action-link page-action-external" id="action-view-markdown" target="_blank" role="menuitem">
+                        <span class="page-action-icon icon-view"></span>
+                        <span class="page-action-text">
+                            <span class="page-action-label">View as Markdown</span>
+                            <span class="page-action-description">Open raw source</span>
+                        </span>
+                    </a>
+                </li>
+                <div class="page-actions-divider"></div>
+                <li class="page-action-item">
+                    <a href="#" class="page-action-link page-action-external" id="action-open-chatgpt" role="menuitem">
+                        <span class="page-action-icon icon-ai"></span>
+                        <span class="page-action-text">
+                            <span class="page-action-label">Open in ChatGPT</span>
+                            <span class="page-action-description">Ask questions about this page</span>
+                        </span>
+                    </a>
+                </li>
+            </ul>
+            <div class="page-actions-footer">ESC to close</div>
+        `;
+
+        // Create a wrapper for button and dropdown
+        const wrapper = document.createElement('div');
+        wrapper.className = 'page-actions-wrapper';
+        wrapper.appendChild(button);
+        wrapper.appendChild(dropdown);
+
+        // Inject into main content area
+        mainContent.appendChild(wrapper);
+
+        // Append overlay to body
+        document.body.appendChild(overlay);
+
+        return { button, dropdown, overlay, wrapper };
+    }
+
+    // Toggle dropdown
+    function toggleDropdown(button, dropdown, overlay) {
+        const isActive = dropdown.classList.contains('active');
+
+        if (isActive) {
+            closeDropdown(button, dropdown, overlay);
+        } else {
+            openDropdown(button, dropdown, overlay);
+        }
+    }
+
+    function openDropdown(button, dropdown, overlay) {
+        dropdown.classList.add('active');
+        // Don't activate overlay - not needed
+        button.classList.add('active');
+        button.setAttribute('aria-expanded', 'true');
+    }
+
+    function closeDropdown(button, dropdown, overlay) {
+        dropdown.classList.remove('active');
+        // Don't deactivate overlay - not needed
+        button.classList.remove('active');
+        button.setAttribute('aria-expanded', 'false');
+    }
+
+    // Show notification
+    function showNotification(message, duration = 2000) {
+        const notification = document.createElement('div');
+        notification.className = 'page-actions-notification';
+        notification.textContent = message;
+        document.body.appendChild(notification);
+
+        setTimeout(() => {
+            notification.remove();
+        }, duration);
+    }
+
+    // Copy markdown to clipboard
+    async function copyMarkdownToClipboard(link) {
+        // Add loading state
+        link.classList.add('loading');
+
+        try {
+            const markdown = await getMarkdownContent();
+
+            // Copy to clipboard
+            await navigator.clipboard.writeText(markdown);
+
+            // Visual feedback
+            link.classList.remove('loading');
+            link.classList.add('page-action-copied');
+
+            showNotification('Markdown copied to clipboard!');
+
+            // Reset after delay
+            setTimeout(() => {
+                link.classList.remove('page-action-copied');
+            }, 2000);
+
+        } catch (error) {
+            console.error('Error copying markdown:', error);
+            link.classList.remove('loading');
+            showNotification('Error: Could not copy markdown');
+        }
+    }
+
+    // View markdown in new tab
+    function viewMarkdown() {
+        const githubUrl = getGithubFileUrl();
+        window.open(githubUrl, '_blank', 'noopener,noreferrer');
+    }
+
+    function getCurrentPageUrl() {
+        const { href } = window.location;
+        return href.split('#')[0];
+    }
+
+    function openChatGPT() {
+        const pageUrl = getCurrentPageUrl();
+        const prompt = encodeURIComponent(`Read ${pageUrl} so I can ask questions about it.`);
+        const chatUrl = `https://chatgpt.com/?hint=search&prompt=${prompt}`;
+        window.open(chatUrl, '_blank', 'noopener,noreferrer');
+    }
+
+    (async () => {
+        if (!shouldShowButton()) {
+            return;
+        }
+
+        const markdownAvailable = await ensureMarkdownCached();
+        if (!markdownAvailable) {
+            return;
+        }
+
+        const ui = createPageActionsUI();
+        if (!ui) {
+            return;
+        }
+
+        const { button, dropdown, overlay } = ui;
+
+        // Event listeners
+        button.addEventListener('click', (e) => {
+            e.stopPropagation();
+            toggleDropdown(button, dropdown, overlay);
+        });
+
+        overlay.addEventListener('click', () => {
+            closeDropdown(button, dropdown, overlay);
+        });
+
+        // Copy markdown action
+        document.getElementById('action-copy-markdown').addEventListener('click', async (e) => {
+            e.preventDefault();
+            e.stopPropagation();
+            await copyMarkdownToClipboard(e.currentTarget);
+        });
+
+        // View markdown action
+        document.getElementById('action-view-markdown').addEventListener('click', (e) => {
+            e.preventDefault();
+            e.stopPropagation();
+            viewMarkdown();
+            closeDropdown(button, dropdown, overlay);
+        });
+
+        // Open in ChatGPT action
+        document.getElementById('action-open-chatgpt').addEventListener('click', (e) => {
+            e.preventDefault();
+            e.stopPropagation();
+            openChatGPT();
+            closeDropdown(button, dropdown, overlay);
+        });
+
+        // Close on ESC key
+        document.addEventListener('keydown', (e) => {
+            if (e.key === 'Escape' && dropdown.classList.contains('active')) {
+                closeDropdown(button, dropdown, overlay);
+            }
+        });
+
+        // Close when clicking outside
+        document.addEventListener('click', (e) => {
+            if (!dropdown.contains(e.target) && !button.contains(e.target)) {
+                closeDropdown(button, dropdown, overlay);
+            }
+        });
+
+        // Prevent dropdown from closing when clicking inside
+        dropdown.addEventListener('click', (e) => {
+            // Only stop propagation if not clicking on a link
+            if (!e.target.closest('.page-action-link')) {
+                e.stopPropagation();
+            }
+        });
+
+        // Close dropdown on link click (except for copy which handles itself)
+        dropdown.querySelectorAll('.page-action-link:not(#action-copy-markdown)').forEach(link => {
+            link.addEventListener('click', () => {
+                if (!link.classList.contains('disabled')) {
+                    setTimeout(() => {
+                        closeDropdown(button, dropdown, overlay);
+                    }, 100);
+                }
+            });
+        });
+
+        // Handle window resize
+        let resizeTimer;
+        window.addEventListener('resize', () => {
+            clearTimeout(resizeTimer);
+            resizeTimer = setTimeout(() => {
+                // Close dropdown on resize to prevent positioning issues
+                if (dropdown.classList.contains('active')) {
+                    closeDropdown(button, dropdown, overlay);
+                }
+            }, 250);
+        });
+
+        // Accessibility: Focus management
+        button.addEventListener('keydown', (e) => {
+            if (e.key === 'Enter' || e.key === ' ') {
+                e.preventDefault();
+                toggleDropdown(button, dropdown, overlay);
+
+                // Focus first menu item when opening
+                if (dropdown.classList.contains('active')) {
+                    const firstLink = dropdown.querySelector('.page-action-link:not(.disabled)');
+                    if (firstLink) {
+                        setTimeout(() => firstLink.focus(), 100);
+                    }
+                }
+            }
+        });
+
+        // Arrow key navigation within menu
+        dropdown.addEventListener('keydown', (e) => {
+            if (!dropdown.classList.contains('active')) return;
+
+            const links = Array.from(dropdown.querySelectorAll('.page-action-link:not(.disabled)'));
+            const currentIndex = links.indexOf(document.activeElement);
+
+            if (e.key === 'ArrowDown') {
+                e.preventDefault();
+                const nextIndex = (currentIndex + 1) % links.length;
+                links[nextIndex].focus();
+            } else if (e.key === 'ArrowUp') {
+                e.preventDefault();
+                const prevIndex = (currentIndex - 1 + links.length) % links.length;
+                links[prevIndex].focus();
+            } else if (e.key === 'Home') {
+                e.preventDefault();
+                links[0].focus();
+            } else if (e.key === 'End') {
+                e.preventDefault();
+                links[links.length - 1].focus();
+            }
+        });
+
+        console.log('Page Actions initialized for:', getCurrentMarkdownPath());
+    })();
+});

docs/md_v2/core/docker-deployment.md

@@ -6,18 +6,6 @@
   - [Option 1: Using Pre-built Docker Hub Images (Recommended)](#option-1-using-pre-built-docker-hub-images-recommended)
   - [Option 2: Using Docker Compose](#option-2-using-docker-compose)
   - [Option 3: Manual Local Build & Run](#option-3-manual-local-build--run)
-- [Dockerfile Parameters](#dockerfile-parameters)
-- [Using the API](#using-the-api)
-  - [Playground Interface](#playground-interface)
-  - [Python SDK](#python-sdk)
-  - [Understanding Request Schema](#understanding-request-schema)
-  - [REST API Examples](#rest-api-examples)
-- [Additional API Endpoints](#additional-api-endpoints)
-  - [HTML Extraction Endpoint](#html-extraction-endpoint)
-  - [Screenshot Endpoint](#screenshot-endpoint)
-  - [PDF Export Endpoint](#pdf-export-endpoint)
-  - [JavaScript Execution Endpoint](#javascript-execution-endpoint)
-  - [Library Context Endpoint](#library-context-endpoint)
 - [MCP (Model Context Protocol) Support](#mcp-model-context-protocol-support)
   - [What is MCP?](#what-is-mcp)
   - [Connecting via MCP](#connecting-via-mcp)
@@ -25,9 +13,28 @@
   - [Available MCP Tools](#available-mcp-tools)
   - [Testing MCP Connections](#testing-mcp-connections)
   - [MCP Schemas](#mcp-schemas)
+- [Additional API Endpoints](#additional-api-endpoints)
+  - [HTML Extraction Endpoint](#html-extraction-endpoint)
+  - [Screenshot Endpoint](#screenshot-endpoint)
+  - [PDF Export Endpoint](#pdf-export-endpoint)
+  - [JavaScript Execution Endpoint](#javascript-execution-endpoint)
+- [User-Provided Hooks API](#user-provided-hooks-api)
+  - [Hook Information Endpoint](#hook-information-endpoint)
+  - [Available Hook Points](#available-hook-points)
+  - [Using Hooks in Requests](#using-hooks-in-requests)
+  - [Hook Examples with Real URLs](#hook-examples-with-real-urls)
+  - [Security Best Practices](#security-best-practices)
+  - [Hook Response Information](#hook-response-information)
+  - [Error Handling](#error-handling)
+  - [Hooks Utility: Function-Based Approach (Python)](#hooks-utility-function-based-approach-python)
+- [Dockerfile Parameters](#dockerfile-parameters)
+- [Using the API](#using-the-api)
+  - [Playground Interface](#playground-interface)
+  - [Python SDK](#python-sdk)
+  - [Understanding Request Schema](#understanding-request-schema)
+  - [REST API Examples](#rest-api-examples)
+  - [LLM Configuration Examples](#llm-configuration-examples)
 - [Metrics & Monitoring](#metrics--monitoring)
-- [Deployment Scenarios](#deployment-scenarios)
-- [Complete Examples](#complete-examples)
 - [Server Configuration](#server-configuration)
   - [Understanding config.yml](#understanding-configyml)
   - [JWT Authentication](#jwt-authentication)
@@ -832,6 +839,275 @@ else:
 
 > 💡 **Remember**: Always test your hooks on safe, known websites first before using them on production sites. Never crawl sites that you don't have permission to access or that might be malicious.
 
+### Hooks Utility: Function-Based Approach (Python)
+
+For Python developers, Crawl4AI provides a more convenient way to work with hooks using the `hooks_to_string()` utility function and Docker client integration.
+
+#### Why Use Function-Based Hooks?
+
+**String-Based Approach (shown above)**:
+```python
+hooks_code = {
+    "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    return page
+"""
+}
+```
+
+**Function-Based Approach (recommended for Python)**:
+```python
+from crawl4ai import Crawl4aiDockerClient
+
+async def my_hook(page, context, **kwargs):
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    return page
+
+async with Crawl4aiDockerClient(base_url="http://localhost:11235") as client:
+    result = await client.crawl(
+        ["https://example.com"],
+        hooks={"on_page_context_created": my_hook}
+    )
+```
+
+**Benefits**:
+- ✅ Write hooks as regular Python functions
+- ✅ Full IDE support (autocomplete, syntax highlighting, type checking)
+- ✅ Easy to test and debug
+- ✅ Reusable hook libraries
+- ✅ Automatic conversion to API format
+
+#### Using the Hooks Utility
+
+The `hooks_to_string()` utility converts Python function objects to the string format required by the API:
+
+```python
+from crawl4ai import hooks_to_string
+
+# Define your hooks as functions
+async def setup_hook(page, context, **kwargs):
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    await context.add_cookies([{
+        "name": "session",
+        "value": "token",
+        "domain": ".example.com"
+    }])
+    return page
+
+async def scroll_hook(page, context, **kwargs):
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    return page
+
+# Convert to string format
+hooks_dict = {
+    "on_page_context_created": setup_hook,
+    "before_retrieve_html": scroll_hook
+}
+hooks_string = hooks_to_string(hooks_dict)
+
+# Now use with REST API or Docker client
+# hooks_string contains the string representations
+```
+
+#### Docker Client with Automatic Conversion
+
+The Docker client automatically detects and converts function objects:
+
+```python
+from crawl4ai import Crawl4aiDockerClient
+
+async def auth_hook(page, context, **kwargs):
+    """Add authentication cookies"""
+    await context.add_cookies([{
+        "name": "auth_token",
+        "value": "your_token",
+        "domain": ".example.com"
+    }])
+    return page
+
+async def performance_hook(page, context, **kwargs):
+    """Block unnecessary resources"""
+    await context.route("**/*.{png,jpg,gif}", lambda r: r.abort())
+    await context.route("**/analytics/*", lambda r: r.abort())
+    return page
+
+async with Crawl4aiDockerClient(base_url="http://localhost:11235") as client:
+    # Pass functions directly - automatic conversion!
+    result = await client.crawl(
+        ["https://example.com"],
+        hooks={
+            "on_page_context_created": performance_hook,
+            "before_goto": auth_hook
+        },
+        hooks_timeout=30  # Optional timeout in seconds (1-120)
+    )
+
+    print(f"Success: {result.success}")
+    print(f"HTML: {len(result.html)} chars")
+```
+
+#### Creating Reusable Hook Libraries
+
+Build collections of reusable hooks:
+
+```python
+# hooks_library.py
+class CrawlHooks:
+    """Reusable hook collection for common crawling tasks"""
+
+    @staticmethod
+    async def block_images(page, context, **kwargs):
+        """Block all images to speed up crawling"""
+        await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda r: r.abort())
+        return page
+
+    @staticmethod
+    async def block_analytics(page, context, **kwargs):
+        """Block analytics and tracking scripts"""
+        tracking_domains = [
+            "**/google-analytics.com/*",
+            "**/googletagmanager.com/*",
+            "**/facebook.com/tr/*",
+            "**/doubleclick.net/*"
+        ]
+        for domain in tracking_domains:
+            await context.route(domain, lambda r: r.abort())
+        return page
+
+    @staticmethod
+    async def scroll_infinite(page, context, **kwargs):
+        """Handle infinite scroll to load more content"""
+        previous_height = 0
+        for i in range(5):  # Max 5 scrolls
+            current_height = await page.evaluate("document.body.scrollHeight")
+            if current_height == previous_height:
+                break
+            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+            await page.wait_for_timeout(1000)
+            previous_height = current_height
+        return page
+
+    @staticmethod
+    async def wait_for_dynamic_content(page, context, url, response, **kwargs):
+        """Wait for dynamic content to load"""
+        await page.wait_for_timeout(2000)
+        try:
+            # Click "Load More" if present
+            load_more = await page.query_selector('[class*="load-more"]')
+            if load_more:
+                await load_more.click()
+                await page.wait_for_timeout(1000)
+        except:
+            pass
+        return page
+
+# Use in your application
+from hooks_library import CrawlHooks
+from crawl4ai import Crawl4aiDockerClient
+
+async def crawl_with_optimizations(url):
+    async with Crawl4aiDockerClient() as client:
+        result = await client.crawl(
+            [url],
+            hooks={
+                "on_page_context_created": CrawlHooks.block_images,
+                "before_retrieve_html": CrawlHooks.scroll_infinite
+            }
+        )
+        return result
+```
+
+#### Choosing the Right Approach
+
+| Approach | Best For | IDE Support | Language |
+|----------|----------|-------------|----------|
+| **String-based** | Non-Python clients, REST APIs, other languages | ❌ None | Any |
+| **Function-based** | Python applications, local development | ✅ Full | Python only |
+| **Docker Client** | Python apps with automatic conversion | ✅ Full | Python only |
+
+**Recommendation**:
+- **Python applications**: Use Docker client with function objects (easiest)
+- **Non-Python or REST API**: Use string-based hooks (most flexible)
+- **Manual control**: Use `hooks_to_string()` utility (middle ground)
+
+#### Complete Example with Function Hooks
+
+```python
+from crawl4ai import Crawl4aiDockerClient, BrowserConfig, CrawlerRunConfig, CacheMode
+
+# Define hooks as regular Python functions
+async def setup_environment(page, context, **kwargs):
+    """Setup crawling environment"""
+    # Set viewport
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+
+    # Block resources for speed
+    await context.route("**/*.{png,jpg,gif}", lambda r: r.abort())
+
+    # Add custom headers
+    await page.set_extra_http_headers({
+        "Accept-Language": "en-US",
+        "X-Custom-Header": "Crawl4AI"
+    })
+
+    print("[HOOK] Environment configured")
+    return page
+
+async def extract_content(page, context, **kwargs):
+    """Extract and prepare content"""
+    # Scroll to load lazy content
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    await page.wait_for_timeout(1000)
+
+    # Extract metadata
+    metadata = await page.evaluate('''() => ({
+        title: document.title,
+        links: document.links.length,
+        images: document.images.length
+    })''')
+
+    print(f"[HOOK] Page metadata: {metadata}")
+    return page
+
+async def main():
+    async with Crawl4aiDockerClient(base_url="http://localhost:11235", verbose=True) as client:
+        # Configure crawl
+        browser_config = BrowserConfig(headless=True)
+        crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+
+        # Crawl with hooks
+        result = await client.crawl(
+            ["https://httpbin.org/html"],
+            browser_config=browser_config,
+            crawler_config=crawler_config,
+            hooks={
+                "on_page_context_created": setup_environment,
+                "before_retrieve_html": extract_content
+            },
+            hooks_timeout=30
+        )
+
+        if result.success:
+            print(f"✅ Crawl successful!")
+            print(f"   URL: {result.url}")
+            print(f"   HTML: {len(result.html)} chars")
+            print(f"   Markdown: {len(result.markdown)} chars")
+        else:
+            print(f"❌ Crawl failed: {result.error_message}")
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())
+```
+
+#### Additional Resources
+
+- **Comprehensive Examples**: See `/docs/examples/hooks_docker_client_example.py` for Python function-based examples
+- **REST API Examples**: See `/docs/examples/hooks_rest_api_example.py` for string-based examples
+- **Comparison Guide**: See `/docs/examples/README_HOOKS.md` for detailed comparison
+- **Utility Documentation**: See `/docs/hooks-utility-guide.md` for complete guide
+
 ---
 
 ## Dockerfile Parameters
@@ -892,10 +1168,12 @@ This is the easiest way to translate Python configuration to JSON requests when
 
 Install the SDK: `pip install crawl4ai`
 
+The Python SDK provides a convenient way to interact with the Docker API, including **automatic hook conversion** when using function objects.
+
 ```python
 import asyncio
 from crawl4ai.docker_client import Crawl4aiDockerClient
-from crawl4ai import BrowserConfig, CrawlerRunConfig, CacheMode # Assuming you have crawl4ai installed
+from crawl4ai import BrowserConfig, CrawlerRunConfig, CacheMode
 
 async def main():
     # Point to the correct server port
@@ -907,23 +1185,22 @@ async def main():
         print("--- Running Non-Streaming Crawl ---")
         results = await client.crawl(
             ["https://httpbin.org/html"],
-            browser_config=BrowserConfig(headless=True), # Use library classes for config aid
+            browser_config=BrowserConfig(headless=True),
             crawler_config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
         )
-        if results: # client.crawl returns None on failure
-          print(f"Non-streaming results success: {results.success}")
-          if results.success:
-              for result in results: # Iterate through the CrawlResultContainer
-                  print(f"URL: {result.url}, Success: {result.success}")
+        if results:
+            print(f"Non-streaming results success: {results.success}")
+            if results.success:
+                for result in results:
+                    print(f"URL: {result.url}, Success: {result.success}")
         else:
             print("Non-streaming crawl failed.")
 
-
         # Example Streaming crawl
         print("\n--- Running Streaming Crawl ---")
         stream_config = CrawlerRunConfig(stream=True, cache_mode=CacheMode.BYPASS)
         try:
-            async for result in await client.crawl( # client.crawl returns an async generator for streaming
+            async for result in await client.crawl(
                 ["https://httpbin.org/html", "https://httpbin.org/links/5/0"],
                 browser_config=BrowserConfig(headless=True),
                 crawler_config=stream_config
@@ -932,17 +1209,56 @@ async def main():
         except Exception as e:
             print(f"Streaming crawl failed: {e}")
 
+        # Example with hooks (Python function objects)
+        print("\n--- Crawl with Hooks ---")
+
+        async def my_hook(page, context, **kwargs):
+            """Custom hook to optimize performance"""
+            await page.set_viewport_size({"width": 1920, "height": 1080})
+            await context.route("**/*.{png,jpg}", lambda r: r.abort())
+            print("[HOOK] Page optimized")
+            return page
+
+        result = await client.crawl(
+            ["https://httpbin.org/html"],
+            browser_config=BrowserConfig(headless=True),
+            crawler_config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS),
+            hooks={"on_page_context_created": my_hook},  # Pass function directly!
+            hooks_timeout=30
+        )
+        print(f"Crawl with hooks success: {result.success}")
 
         # Example Get schema
         print("\n--- Getting Schema ---")
         schema = await client.get_schema()
-        print(f"Schema received: {bool(schema)}") # Print whether schema was received
+        print(f"Schema received: {bool(schema)}")
 
 if __name__ == "__main__":
     asyncio.run(main())
 ```
 
-*(SDK parameters like timeout, verify_ssl etc. remain the same)*
+#### SDK Parameters
+
+The Docker client supports the following parameters:
+
+**Client Initialization**:
+- `base_url` (str): URL of the Docker server (default: `http://localhost:8000`)
+- `timeout` (float): Request timeout in seconds (default: 30.0)
+- `verify_ssl` (bool): Verify SSL certificates (default: True)
+- `verbose` (bool): Enable verbose logging (default: True)
+- `log_file` (Optional[str]): Path to log file (default: None)
+
+**crawl() Method**:
+- `urls` (List[str]): List of URLs to crawl
+- `browser_config` (Optional[BrowserConfig]): Browser configuration
+- `crawler_config` (Optional[CrawlerRunConfig]): Crawler configuration
+- `hooks` (Optional[Dict]): Hook functions or strings - **automatically converts function objects!**
+- `hooks_timeout` (int): Timeout for each hook execution in seconds (default: 30)
+
+**Returns**:
+- Single URL: `CrawlResult` object
+- Multiple URLs: `List[CrawlResult]`
+- Streaming: `AsyncGenerator[CrawlResult]`
 
 ### Second Approach: Direct API Calls
 
@@ -1352,19 +1668,40 @@ We're here to help you succeed with Crawl4AI! Here's how to get support:
 
 In this guide, we've covered everything you need to get started with Crawl4AI's Docker deployment:
 - Building and running the Docker container
-- Configuring the environment  
+- Configuring the environment
 - Using the interactive playground for testing
 - Making API requests with proper typing
-- Using the Python SDK
+- Using the Python SDK with **automatic hook conversion**
+- **Working with hooks** - both string-based (REST API) and function-based (Python SDK)
 - Leveraging specialized endpoints for screenshots, PDFs, and JavaScript execution
 - Connecting via the Model Context Protocol (MCP)
 - Monitoring your deployment
 
-The new playground interface at `http://localhost:11235/playground` makes it much easier to test configurations and generate the corresponding JSON for API requests.
+### Key Features
 
-For AI application developers, the MCP integration allows tools like Claude Code to directly access Crawl4AI's capabilities without complex API handling.
+**Hooks Support**: Crawl4AI offers two approaches for working with hooks:
+- **String-based** (REST API): Works with any language, requires manual string formatting
+- **Function-based** (Python SDK): Write hooks as regular Python functions with full IDE support and automatic conversion
 
-Remember, the examples in the `examples` folder are your friends - they show real-world usage patterns that you can adapt for your needs.
+**Playground Interface**: The built-in playground at `http://localhost:11235/playground` makes it easy to test configurations and generate corresponding JSON for API requests.
+
+**MCP Integration**: For AI application developers, the MCP integration allows tools like Claude Code to directly access Crawl4AI's capabilities without complex API handling.
+
+### Next Steps
+
+1. **Explore Examples**: Check out the comprehensive examples in:
+   - `/docs/examples/hooks_docker_client_example.py` - Python function-based hooks
+   - `/docs/examples/hooks_rest_api_example.py` - REST API string-based hooks
+   - `/docs/examples/README_HOOKS.md` - Comparison and guide
+
+2. **Read Documentation**:
+   - `/docs/hooks-utility-guide.md` - Complete hooks utility guide
+   - API documentation for detailed configuration options
+
+3. **Join the Community**:
+   - GitHub: Report issues and contribute
+   - Discord: Get help and share your experiences
+   - Documentation: Comprehensive guides and tutorials
 
 Keep exploring, and don't hesitate to reach out if you need help! We're building something amazing together. 🚀
 

docs/md_v2/core/link-analysis.md

@@ -1,523 +0,0 @@
-# Link Analysis and Scoring
-
-## Introduction
-
-**Link Analysis** is a powerful feature that extracts, analyzes, and scores all links found on a webpage. This endpoint helps you understand the link structure, identify high-value links, and get insights into the connectivity patterns of any website.
-
-Think of it as a smart link discovery tool that not only extracts links but also evaluates their importance, relevance, and quality through advanced scoring algorithms.
-
-## Key Concepts
-
-### What Link Analysis Does
-
-When you analyze a webpage, the system:
-
-1. **Extracts All Links** - Finds every hyperlink on the page
-2. **Scores Links** - Assigns relevance scores based on multiple factors
-3. **Categorizes Links** - Groups links by type (internal, external, etc.)
-4. **Provides Metadata** - URL text, attributes, and context information
-5. **Ranks by Importance** - Orders links from most to least valuable
-
-### Scoring Factors
-
-The link scoring algorithm considers:
-
-- **Text Content**: Link anchor text relevance and descriptiveness
-- **URL Structure**: Depth, parameters, and path patterns
-- **Context**: Surrounding text and page position
-- **Attributes**: Title, rel attributes, and other metadata
-- **Link Type**: Internal vs external classification
-
-## Quick Start
-
-### Basic Usage
-
-```python
-import requests
-
-# Analyze links on a webpage
-response = requests.post(
-    "http://localhost:8000/links/analyze",
-    headers={"Authorization": "Bearer YOUR_TOKEN"},
-    json={
-        "url": "https://example.com"
-    }
-)
-
-result = response.json()
-print(f"Found {len(result.get('internal', []))} internal links")
-print(f"Found {len(result.get('external', []))} external links")
-
-# Show top 3 links by score
-for link_type in ['internal', 'external']:
-    if link_type in result:
-        top_links = sorted(result[link_type], key=lambda x: x.get('score', 0), reverse=True)[:3]
-        print(f"\nTop {link_type} links:")
-        for link in top_links:
-            print(f"- {link.get('url', 'N/A')} (score: {link.get('score', 0):.2f})")
-```
-
-### With Custom Configuration
-
-```python
-response = requests.post(
-    "http://localhost:8000/links/analyze",
-    headers={"Authorization": "Bearer YOUR_TOKEN"},
-    json={
-        "url": "https://news.example.com",
-        "config": {
-            "force": False,           # Skip cache
-            "wait_for": 2.0,          # Wait for dynamic content
-            "simulate_user": True,     # User-like browsing
-            "override_navigator": True # Custom user agent
-        }
-    }
-)
-```
-
-## Configuration Options
-
-The `config` parameter accepts a `LinkPreviewConfig` dictionary:
-
-### Basic Options
-
-```python
-config = {
-    "force": False,                    # Force fresh crawl (default: False)
-    "wait_for": None,                  # CSS selector or timeout in seconds
-    "simulate_user": True,             # Simulate human behavior
-    "override_navigator": True,        # Override browser navigator
-    "headers": {                       # Custom headers
-        "Accept-Language": "en-US,en;q=0.9"
-    }
-}
-```
-
-### Advanced Options
-
-```python
-config = {
-    # Timing and behavior
-    "delay_before_return_html": 0.5,   # Delay before HTML extraction
-    "js_code": ["window.scrollTo(0, document.body.scrollHeight)"],  # JS to execute
-
-    # Content processing
-    "word_count_threshold": 1,         # Minimum word count
-    "exclusion_patterns": [            # Link patterns to exclude
-        r".*/logout.*",
-        r".*/admin.*"
-    ],
-
-    # Caching and session
-    "session_id": "my-session-123",    # Session identifier
-    "magic": False                     # Magic link processing
-}
-```
-
-## Response Structure
-
-The endpoint returns a JSON object with categorized links:
-
-```json
-{
-  "internal": [
-    {
-      "url": "https://example.com/about",
-      "text": "About Us",
-      "title": "Learn about our company",
-      "score": 0.85,
-      "context": "footer navigation",
-      "attributes": {
-        "rel": ["nofollow"],
-        "target": "_blank"
-      }
-    }
-  ],
-  "external": [
-    {
-      "url": "https://partner-site.com",
-      "text": "Partner Site",
-      "title": "Visit our partner",
-      "score": 0.72,
-      "context": "main content",
-      "attributes": {}
-    }
-  ],
-  "social": [...],
-  "download": [...],
-  "email": [...],
-  "phone": [...]
-}
-```
-
-### Link Categories
-
-| Category | Description | Example |
-|----------|-------------|---------|
-| **internal** | Links within the same domain | `/about`, `https://example.com/contact` |
-| **external** | Links to different domains | `https://google.com` |
-| **social** | Social media platform links | `https://twitter.com/user` |
-| **download** | File download links | `/files/document.pdf` |
-| **email** | Email addresses | `mailto:contact@example.com` |
-| **phone** | Phone numbers | `tel:+1234567890` |
-
-### Link Metadata
-
-Each link object contains:
-
-```python
-{
-    "url": str,           # The actual href value
-    "text": str,          # Anchor text content
-    "title": str,         # Title attribute (if any)
-    "score": float,       # Relevance score (0.0-1.0)
-    "context": str,       # Where the link was found
-    "attributes": dict,   # All HTML attributes
-    "hash": str,          # URL fragment (if any)
-    "domain": str,        # Extracted domain name
-    "scheme": str,        # URL scheme (http/https/etc)
-}
-```
-
-## Practical Examples
-
-### SEO Audit Tool
-
-```python
-def seo_audit(url: str):
-    """Perform SEO link analysis on a webpage"""
-    response = requests.post(
-        "http://localhost:8000/links/analyze",
-        headers={"Authorization": "Bearer YOUR_TOKEN"},
-        json={"url": url}
-    )
-
-    result = response.json()
-
-    print(f"📊 SEO Audit for {url}")
-    print(f"Internal links: {len(result.get('internal', []))}")
-    print(f"External links: {len(result.get('external', []))}")
-
-    # Check for SEO issues
-    internal_links = result.get('internal', [])
-    external_links = result.get('external', [])
-
-    # Find links with low scores
-    low_score_links = [link for link in internal_links if link.get('score', 0) < 0.3]
-    if low_score_links:
-        print(f"⚠️  Found {len(low_score_links)} low-quality internal links")
-
-    # Find external opportunities
-    high_value_external = [link for link in external_links if link.get('score', 0) > 0.7]
-    if high_value_external:
-        print(f"✅ Found {len(high_value_external)} high-value external links")
-
-    return result
-
-# Usage
-audit_result = seo_audit("https://example.com")
-```
-
-### Competitor Analysis
-
-```python
-def competitor_analysis(urls: list):
-    """Analyze link patterns across multiple competitor sites"""
-    all_results = {}
-
-    for url in urls:
-        response = requests.post(
-            "http://localhost:8000/links/analyze",
-            headers={"Authorization": "Bearer YOUR_TOKEN"},
-            json={"url": url}
-        )
-        all_results[url] = response.json()
-
-    # Compare external link strategies
-    print("🔍 Competitor Link Analysis")
-    for url, result in all_results.items():
-        external_links = result.get('external', [])
-        avg_score = sum(link.get('score', 0) for link in external_links) / len(external_links) if external_links else 0
-        print(f"{url}: {len(external_links)} external links (avg score: {avg_score:.2f})")
-
-    return all_results
-
-# Usage
-competitors = [
-    "https://competitor1.com",
-    "https://competitor2.com",
-    "https://competitor3.com"
-]
-analysis = competitor_analysis(competitors)
-```
-
-### Content Discovery
-
-```python
-def discover_related_content(start_url: str, max_depth: int = 2):
-    """Discover related content through link analysis"""
-    visited = set()
-    queue = [(start_url, 0)]
-
-    while queue and len(visited) < 20:
-        current_url, depth = queue.pop(0)
-
-        if current_url in visited or depth > max_depth:
-            continue
-
-        visited.add(current_url)
-
-        try:
-            response = requests.post(
-                "http://localhost:8000/links/analyze",
-                headers={"Authorization": "Bearer YOUR_TOKEN"},
-                json={"url": current_url}
-            )
-
-            result = response.json()
-            internal_links = result.get('internal', [])
-
-            # Sort by score and add top links to queue
-            top_links = sorted(internal_links, key=lambda x: x.get('score', 0), reverse=True)[:3]
-
-            for link in top_links:
-                if link['url'] not in visited:
-                    queue.append((link['url'], depth + 1))
-                    print(f"🔗 Found: {link['text']} ({link['score']:.2f})")
-
-        except Exception as e:
-            print(f"❌ Error analyzing {current_url}: {e}")
-
-    return visited
-
-# Usage
-related_pages = discover_related_content("https://blog.example.com")
-print(f"Discovered {len(related_pages)} related pages")
-```
-
-## Best Practices
-
-### 1. Request Optimization
-
-```python
-# ✅ Good: Use appropriate timeouts
-response = requests.post(
-    "http://localhost:8000/links/analyze",
-    headers={"Authorization": "Bearer YOUR_TOKEN"},
-    json={"url": url},
-    timeout=30  # 30 second timeout
-)
-
-# ✅ Good: Configure wait times for dynamic sites
-config = {
-    "wait_for": 2.0,  # Wait for JavaScript to load
-    "simulate_user": True
-}
-```
-
-### 2. Error Handling
-
-```python
-def safe_link_analysis(url: str):
-    try:
-        response = requests.post(
-            "http://localhost:8000/links/analyze",
-            headers={"Authorization": "Bearer YOUR_TOKEN"},
-            json={"url": url},
-            timeout=30
-        )
-
-        if response.status_code == 200:
-            return response.json()
-        elif response.status_code == 400:
-            print("❌ Invalid request format")
-        elif response.status_code == 500:
-            print("❌ Server error during analysis")
-        else:
-            print(f"❌ Unexpected status code: {response.status_code}")
-
-    except requests.Timeout:
-        print("⏰ Request timed out")
-    except requests.ConnectionError:
-        print("🔌 Connection error")
-    except Exception as e:
-        print(f"❌ Unexpected error: {e}")
-
-    return None
-```
-
-### 3. Data Processing
-
-```python
-def process_links_data(result: dict):
-    """Process and filter link analysis results"""
-
-    # Filter by minimum score
-    min_score = 0.5
-    high_quality_links = {}
-
-    for category, links in result.items():
-        filtered_links = [
-            link for link in links
-            if link.get('score', 0) >= min_score
-        ]
-        if filtered_links:
-            high_quality_links[category] = filtered_links
-
-    # Extract unique domains
-    domains = set()
-    for links in result.get('external', []):
-        domains.add(links.get('domain', ''))
-
-    return {
-        'filtered_links': high_quality_links,
-        'unique_domains': list(domains),
-        'total_links': sum(len(links) for links in result.values())
-    }
-```
-
-## Performance Considerations
-
-### Response Times
-
-- **Simple pages**: 2-5 seconds
-- **Complex pages**: 5-15 seconds
-- **JavaScript-heavy**: 10-30 seconds
-
-### Rate Limiting
-
-The endpoint includes built-in rate limiting. For bulk analysis:
-
-```python
-import time
-
-def bulk_link_analysis(urls: list, delay: float = 1.0):
-    """Analyze multiple URLs with rate limiting"""
-    results = {}
-
-    for url in urls:
-        result = safe_link_analysis(url)
-        if result:
-            results[url] = result
-
-        # Respect rate limits
-        time.sleep(delay)
-
-    return results
-```
-
-## Error Handling
-
-### Common Errors and Solutions
-
-| Error Code | Cause | Solution |
-|------------|-------|----------|
-| **400** | Invalid URL or config | Check URL format and config structure |
-| **401** | Invalid authentication | Verify your API token |
-| **429** | Rate limit exceeded | Add delays between requests |
-| **500** | Crawl failure | Check if site is accessible |
-| **503** | Service unavailable | Try again later |
-
-### Debug Mode
-
-```python
-# Enable verbose logging for debugging
-config = {
-    "headers": {
-        "User-Agent": "Crawl4AI-Debug/1.0"
-    }
-}
-
-# Include error details in response
-try:
-    response = requests.post(
-        "http://localhost:8000/links/analyze",
-        headers={"Authorization": "Bearer YOUR_TOKEN"},
-        json={"url": url, "config": config}
-    )
-    response.raise_for_status()
-except requests.HTTPError as e:
-    print(f"Error details: {e.response.text}")
-```
-
-## API Reference
-
-### Endpoint Details
-
-- **URL**: `/links/analyze`
-- **Method**: `POST`
-- **Content-Type**: `application/json`
-- **Authentication**: Bearer token required
-
-### Request Schema
-
-```python
-{
-    "url": str,                    # Required: URL to analyze
-    "config": {                    # Optional: LinkPreviewConfig
-        "force": bool,
-        "wait_for": float,
-        "simulate_user": bool,
-        "override_navigator": bool,
-        "headers": dict,
-        "js_code": list,
-        "delay_before_return_html": float,
-        "word_count_threshold": int,
-        "exclusion_patterns": list,
-        "session_id": str,
-        "magic": bool
-    }
-}
-```
-
-### Response Schema
-
-```python
-{
-    "internal": [LinkObject],
-    "external": [LinkObject],
-    "social": [LinkObject],
-    "download": [LinkObject],
-    "email": [LinkObject],
-    "phone": [LinkObject]
-}
-```
-
-### LinkObject Schema
-
-```python
-{
-    "url": str,
-    "text": str,
-    "title": str,
-    "score": float,
-    "context": str,
-    "attributes": dict,
-    "hash": str,
-    "domain": str,
-    "scheme": str
-}
-```
-
-## Next Steps
-
-- Learn about [Advanced Link Processing](../advanced/link-processing.md)
-- Explore the [Link Preview Configuration](../api/link-preview-config.md)
-- See more [Examples](https://github.com/unclecode/crawl4ai/tree/main/docs/examples/link-analysis)
-
-## FAQ
-
-**Q: How is the link score calculated?**
-A: The score considers multiple factors including anchor text relevance, URL structure, page context, and link attributes. Scores range from 0.0 (lowest quality) to 1.0 (highest quality).
-
-**Q: Can I analyze password-protected pages?**
-A: Yes! Use the `js_code` parameter to handle authentication, or include session cookies in the `headers` configuration.
-
-**Q: How many links can I analyze at once?**
-A: There's no hard limit on the number of links per page, but very large pages (>10,000 links) may take longer to process.
-
-**Q: Can I filter out certain types of links?**
-A: Use the `exclusion_patterns` parameter in the config to filter out unwanted links using regex patterns.
-
-**Q: Does this work with JavaScript-heavy sites?**
-A: Absolutely! The crawler waits for JavaScript execution and can even run custom JavaScript using the `js_code` parameter.

docs/md_v2/index.md

@@ -59,6 +59,27 @@ Crawl4AI is the #1 trending GitHub repository, actively maintained by a vibrant
 
 > **Note**: If you're looking for the old documentation, you can access it [here](https://old.docs.crawl4ai.com).
 
+## 🆕 AI Assistant Skill Now Available!
+
+<div style="background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); padding: 20px; border-radius: 10px; margin: 20px 0; box-shadow: 0 4px 6px rgba(0,0,0,0.1);">
+  <h3 style="color: white; margin: 0 0 10px 0;">🤖 Crawl4AI Skill for Claude & AI Assistants</h3>
+  <p style="color: white; margin: 10px 0;">Supercharge your AI coding assistant with complete Crawl4AI knowledge! Download our comprehensive skill package that includes:</p>
+  <ul style="color: white; margin: 10px 0;">
+    <li>📚 Complete SDK reference (23K+ words)</li>
+    <li>🚀 Ready-to-use extraction scripts</li>
+    <li>⚡ Schema generation for efficient scraping</li>
+    <li>🔧 Version 0.7.4 compatible</li>
+  </ul>
+  <div style="text-align: center; margin-top: 15px;">
+    <a href="assets/crawl4ai-skill.zip" download style="background: white; color: #667eea; padding: 12px 30px; border-radius: 5px; text-decoration: none; font-weight: bold; display: inline-block; transition: transform 0.2s;">
+      📦 Download Skill Package
+    </a>
+  </div>
+  <p style="color: white; margin: 15px 0 0 0; font-size: 0.9em; text-align: center;">
+    Works with Claude, Cursor, Windsurf, and other AI coding assistants. Import the .zip file into your AI assistant's skill/knowledge system.
+  </p>
+</div>
+
 ## 🎯 New: Adaptive Web Crawling
 
 Crawl4AI now features intelligent adaptive crawling that knows when to stop! Using advanced information foraging algorithms, it determines when sufficient information has been gathered to answer your query.

docs/md_v2/marketplace/README.md

@@ -0,0 +1,66 @@
+# Crawl4AI Marketplace
+
+A terminal-themed marketplace for tools, integrations, and resources related to Crawl4AI.
+
+## Setup
+
+### Backend
+
+1. Install dependencies:
+```bash
+cd backend
+pip install -r requirements.txt
+```
+
+2. Generate dummy data:
+```bash
+python dummy_data.py
+```
+
+3. Run the server:
+```bash
+python server.py
+```
+
+The API will be available at http://localhost:8100
+
+### Frontend
+
+1. Open `frontend/index.html` in your browser
+2. Or serve via MkDocs as part of the documentation site
+
+## Database Schema
+
+The marketplace uses SQLite with automatic migration from `schema.yaml`. Tables include:
+- **apps**: Tools and integrations
+- **articles**: Reviews, tutorials, and news
+- **categories**: App categories
+- **sponsors**: Sponsored content
+
+## API Endpoints
+
+- `GET /api/apps` - List apps with filters
+- `GET /api/articles` - List articles
+- `GET /api/categories` - Get all categories
+- `GET /api/sponsors` - Get active sponsors
+- `GET /api/search?q=query` - Search across content
+- `GET /api/stats` - Marketplace statistics
+
+## Features
+
+- **Smart caching**: LocalStorage with TTL (1 hour)
+- **Terminal theme**: Consistent with Crawl4AI branding
+- **Responsive design**: Works on all devices
+- **Fast search**: Debounced with 300ms delay
+- **CORS protected**: Only crawl4ai.com and localhost
+
+## Admin Panel
+
+Coming soon - for now, edit the database directly or modify `dummy_data.py`
+
+## Deployment
+
+For production deployment on EC2:
+1. Update `API_BASE` in `marketplace.js` to production URL
+2. Run FastAPI with proper production settings (use gunicorn/uvicorn)
+3. Set up nginx proxy if needed

docs/md_v2/marketplace/admin/admin.css

@@ -0,0 +1,759 @@
+/* Admin Dashboard - C4AI Terminal Style */
+
+/* Utility Classes */
+.hidden {
+    display: none !important;
+}
+
+/* Brand Colors */
+:root {
+    --c4ai-cyan: #50ffff;
+    --c4ai-green: #50ff50;
+    --c4ai-yellow: #ffff50;
+    --c4ai-pink: #ff50ff;
+    --c4ai-blue: #5050ff;
+}
+
+.admin-container {
+    min-height: 100vh;
+    background: var(--bg-dark);
+}
+
+/* Login Screen */
+.login-screen {
+    min-height: 100vh;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    background: linear-gradient(135deg, #070708 0%, #1a1a2e 100%);
+}
+
+.login-box {
+    background: var(--bg-secondary);
+    border: 2px solid var(--primary-cyan);
+    padding: 3rem;
+    width: 400px;
+    box-shadow: 0 0 40px rgba(80, 255, 255, 0.2);
+    text-align: center;
+}
+
+.login-logo {
+    height: 60px;
+    margin-bottom: 2rem;
+    filter: brightness(1.2);
+}
+
+.login-box h1 {
+    color: var(--primary-cyan);
+    font-size: 1.5rem;
+    margin-bottom: 2rem;
+}
+
+#login-form input {
+    width: 100%;
+    padding: 0.75rem;
+    background: var(--bg-dark);
+    border: 1px solid var(--border-color);
+    color: var(--text-primary);
+    font-family: inherit;
+    margin-bottom: 1rem;
+}
+
+#login-form input:focus {
+    outline: none;
+    border-color: var(--primary-cyan);
+}
+
+#login-form button {
+    width: 100%;
+    padding: 0.75rem;
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    border: none;
+    color: var(--bg-dark);
+    font-weight: 600;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+#login-form button:hover {
+    box-shadow: 0 4px 15px rgba(80, 255, 255, 0.3);
+    transform: translateY(-2px);
+}
+
+.error-msg {
+    color: var(--error);
+    font-size: 0.875rem;
+    margin-top: 1rem;
+}
+
+/* Admin Dashboard */
+.admin-dashboard.hidden {
+    display: none;
+}
+
+.admin-header {
+    background: var(--bg-secondary);
+    border-bottom: 2px solid var(--primary-cyan);
+    padding: 1rem 0;
+}
+
+.header-content {
+    max-width: 1800px;
+    margin: 0 auto;
+    padding: 0 2rem;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+}
+
+.header-left {
+    display: flex;
+    align-items: center;
+    gap: 1rem;
+}
+
+.header-logo {
+    height: 35px;
+}
+
+.admin-header h1 {
+    font-size: 1.25rem;
+    color: var(--primary-cyan);
+}
+
+.header-right {
+    display: flex;
+    align-items: center;
+    gap: 2rem;
+}
+
+.admin-user {
+    color: var(--text-secondary);
+}
+
+.logout-btn {
+    padding: 0.5rem 1rem;
+    background: transparent;
+    border: 1px solid var(--error);
+    color: var(--error);
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.logout-btn:hover {
+    background: rgba(255, 60, 116, 0.1);
+}
+
+/* Layout */
+.admin-layout {
+    display: flex;
+    max-width: 1800px;
+    margin: 0 auto;
+    min-height: calc(100vh - 60px);
+}
+
+/* Sidebar */
+.admin-sidebar {
+    width: 250px;
+    background: var(--bg-secondary);
+    border-right: 1px solid var(--border-color);
+    display: flex;
+    flex-direction: column;
+    justify-content: space-between;
+}
+
+.sidebar-nav {
+    padding: 1rem 0;
+}
+
+.nav-btn {
+    width: 100%;
+    padding: 1rem 1.5rem;
+    background: transparent;
+    border: none;
+    border-left: 3px solid transparent;
+    color: var(--text-secondary);
+    text-align: left;
+    cursor: pointer;
+    transition: all 0.2s;
+    display: flex;
+    align-items: center;
+    gap: 0.75rem;
+}
+
+.nav-btn:hover {
+    background: rgba(80, 255, 255, 0.05);
+    color: var(--primary-cyan);
+}
+
+.nav-btn.active {
+    border-left-color: var(--primary-cyan);
+    background: rgba(80, 255, 255, 0.1);
+    color: var(--primary-cyan);
+}
+
+.nav-icon {
+    font-size: 1.25rem;
+    margin-right: 0.25rem;
+    display: inline-block;
+    width: 1.5rem;
+    text-align: center;
+}
+
+.nav-btn[data-section="stats"] .nav-icon {
+    color: var(--c4ai-cyan);
+}
+
+.nav-btn[data-section="apps"] .nav-icon {
+    color: var(--c4ai-green);
+}
+
+.nav-btn[data-section="articles"] .nav-icon {
+    color: var(--c4ai-yellow);
+}
+
+.nav-btn[data-section="categories"] .nav-icon {
+    color: var(--c4ai-pink);
+}
+
+.nav-btn[data-section="sponsors"] .nav-icon {
+    color: var(--c4ai-blue);
+}
+
+.sidebar-actions {
+    padding: 1rem;
+    border-top: 1px solid var(--border-color);
+}
+
+.action-btn {
+    width: 100%;
+    padding: 0.75rem;
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    cursor: pointer;
+    margin-bottom: 0.5rem;
+    transition: all 0.2s;
+}
+
+.action-btn:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+/* Main Content */
+.admin-main {
+    flex: 1;
+    padding: 2rem;
+    overflow-y: auto;
+}
+
+.content-section {
+    display: none;
+}
+
+.content-section.active {
+    display: block;
+}
+
+/* Stats Grid */
+.stats-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+    gap: 1.5rem;
+    margin-bottom: 3rem;
+}
+
+.stat-card {
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.03), rgba(243, 128, 245, 0.02));
+    border: 1px solid rgba(80, 255, 255, 0.3);
+    padding: 1.5rem;
+    display: flex;
+    gap: 1.5rem;
+}
+
+.stat-icon {
+    font-size: 2rem;
+    width: 3rem;
+    height: 3rem;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    border: 2px solid;
+    border-radius: 4px;
+}
+
+.stat-card:nth-child(1) .stat-icon {
+    color: var(--c4ai-cyan);
+    border-color: var(--c4ai-cyan);
+}
+
+.stat-card:nth-child(2) .stat-icon {
+    color: var(--c4ai-green);
+    border-color: var(--c4ai-green);
+}
+
+.stat-card:nth-child(3) .stat-icon {
+    color: var(--c4ai-yellow);
+    border-color: var(--c4ai-yellow);
+}
+
+.stat-card:nth-child(4) .stat-icon {
+    color: var(--c4ai-pink);
+    border-color: var(--c4ai-pink);
+}
+
+.stat-number {
+    font-size: 2rem;
+    color: var(--primary-cyan);
+    font-weight: 600;
+}
+
+.stat-label {
+    color: var(--text-secondary);
+}
+
+.stat-detail {
+    font-size: 0.875rem;
+    color: var(--text-tertiary);
+    margin-top: 0.5rem;
+}
+
+/* Quick Actions */
+.quick-actions {
+    display: flex;
+    gap: 1rem;
+}
+
+.quick-btn {
+    padding: 0.75rem 1.5rem;
+    background: transparent;
+    border: 1px solid var(--primary-cyan);
+    color: var(--primary-cyan);
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.quick-btn:hover {
+    background: rgba(80, 255, 255, 0.1);
+    transform: translateY(-2px);
+}
+
+/* Section Headers */
+.section-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 2rem;
+}
+
+.section-header h2 {
+    font-size: 1.5rem;
+    color: var(--text-primary);
+}
+
+.header-actions {
+    display: flex;
+    gap: 1rem;
+}
+
+.search-input {
+    padding: 0.5rem 1rem;
+    background: var(--bg-dark);
+    border: 1px solid var(--border-color);
+    color: var(--text-primary);
+    width: 250px;
+}
+
+.search-input:focus {
+    outline: none;
+    border-color: var(--primary-cyan);
+}
+
+.filter-select {
+    padding: 0.5rem;
+    background: var(--bg-dark);
+    border: 1px solid var(--border-color);
+    color: var(--text-primary);
+}
+
+.add-btn {
+    padding: 0.5rem 1rem;
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    border: none;
+    color: var(--bg-dark);
+    font-weight: 600;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.add-btn:hover {
+    box-shadow: 0 4px 15px rgba(80, 255, 255, 0.3);
+    transform: translateY(-2px);
+}
+
+/* Data Tables */
+.data-table {
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    overflow-x: auto;
+}
+
+.data-table table {
+    width: 100%;
+    border-collapse: collapse;
+}
+
+.data-table th {
+    background: var(--bg-tertiary);
+    padding: 1rem;
+    text-align: left;
+    color: var(--primary-cyan);
+    font-weight: 600;
+    border-bottom: 2px solid var(--border-color);
+    position: sticky;
+    top: 0;
+    z-index: 10;
+}
+
+.data-table td {
+    padding: 1rem;
+    border-bottom: 1px solid var(--border-color);
+}
+
+.data-table tr:hover {
+    background: rgba(80, 255, 255, 0.03);
+}
+
+/* Table Actions */
+.table-actions {
+    display: flex;
+    gap: 0.5rem;
+}
+
+.table-logo {
+    width: 48px;
+    height: 48px;
+    object-fit: contain;
+    border-radius: 6px;
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    padding: 4px;
+}
+
+.btn-edit, .btn-delete, .btn-duplicate {
+    padding: 0.25rem 0.5rem;
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    cursor: pointer;
+    font-size: 0.875rem;
+}
+
+.btn-edit:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+.btn-delete:hover {
+    border-color: var(--error);
+    color: var(--error);
+}
+
+.btn-duplicate:hover {
+    border-color: var(--accent-pink);
+    color: var(--accent-pink);
+}
+
+/* Badges in Tables */
+.badge {
+    padding: 0.25rem 0.5rem;
+    font-size: 0.75rem;
+    text-transform: uppercase;
+}
+
+.badge.featured {
+    background: var(--primary-cyan);
+    color: var(--bg-dark);
+}
+
+.badge.sponsored {
+    background: var(--warning);
+    color: var(--bg-dark);
+}
+
+.badge.active {
+    background: var(--success);
+    color: var(--bg-dark);
+}
+
+/* Modal Enhancements */
+.modal-content.large {
+    max-width: 1000px;
+    width: 90%;
+    max-height: 90vh;
+}
+
+.modal-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    padding: 1.5rem;
+    border-bottom: 1px solid var(--border-color);
+}
+
+.modal-body {
+    padding: 1.5rem;
+    overflow-y: auto;
+    max-height: calc(90vh - 140px);
+}
+
+.modal-footer {
+    display: flex;
+    justify-content: flex-end;
+    gap: 1rem;
+    padding: 1rem 1.5rem;
+    border-top: 1px solid var(--border-color);
+}
+
+.btn-cancel, .btn-save {
+    padding: 0.5rem 1.5rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.btn-cancel {
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+}
+
+.btn-cancel:hover {
+    border-color: var(--error);
+    color: var(--error);
+}
+
+.btn-save {
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    border: none;
+    color: var(--bg-dark);
+    font-weight: 600;
+}
+
+.btn-save:hover {
+    box-shadow: 0 4px 15px rgba(80, 255, 255, 0.3);
+}
+
+/* Form Styles */
+.form-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
+    gap: 1.5rem;
+}
+
+.form-group {
+    display: flex;
+    flex-direction: column;
+    gap: 0.5rem;
+}
+
+.form-group label {
+    color: var(--text-secondary);
+    font-size: 0.875rem;
+}
+
+.form-group input,
+.form-group select,
+.form-group textarea {
+    padding: 0.5rem;
+    background: var(--bg-dark);
+    border: 1px solid var(--border-color);
+    color: var(--text-primary);
+    font-family: inherit;
+}
+
+.form-group input:focus,
+.form-group select:focus,
+.form-group textarea:focus {
+    outline: none;
+    border-color: var(--primary-cyan);
+}
+
+.form-group.full-width {
+    grid-column: 1 / -1;
+}
+
+.checkbox-group {
+    display: flex;
+    gap: 2rem;
+}
+
+.checkbox-label {
+    display: flex;
+    align-items: center;
+    gap: 0.5rem;
+    cursor: pointer;
+}
+
+.sponsor-form {
+    grid-template-columns: 200px repeat(2, minmax(220px, 1fr));
+    align-items: flex-start;
+    grid-auto-flow: dense;
+}
+
+.sponsor-logo-group {
+    grid-row: span 3;
+    display: flex;
+    flex-direction: column;
+    gap: 0.75rem;
+}
+
+.span-two {
+    grid-column: span 2;
+}
+
+.logo-upload {
+    position: relative;
+    width: 180px;
+}
+
+.image-preview {
+    width: 180px;
+    height: 180px;
+    border: 1px dashed var(--border-color);
+    border-radius: 12px;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    background: var(--bg-tertiary);
+    overflow: hidden;
+}
+
+.image-preview.empty {
+    color: var(--text-secondary);
+    font-size: 0.75rem;
+    text-align: center;
+    padding: 0.75rem;
+}
+
+.image-preview img {
+    max-width: 100%;
+    max-height: 100%;
+    object-fit: contain;
+}
+
+.upload-btn {
+    position: absolute;
+    left: 50%;
+    bottom: 12px;
+    transform: translateX(-50%);
+    padding: 0.35rem 1rem;
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    border: none;
+    border-radius: 999px;
+    color: var(--bg-dark);
+    font-size: 0.75rem;
+    font-weight: 600;
+    cursor: pointer;
+    box-shadow: 0 6px 18px rgba(80, 255, 255, 0.25);
+}
+
+.upload-btn:hover {
+    box-shadow: 0 8px 22px rgba(80, 255, 255, 0.35);
+}
+
+.logo-upload input[type="file"] {
+    display: none;
+}
+
+.upload-hint {
+    font-size: 0.75rem;
+    color: var(--text-secondary);
+    margin: 0;
+}
+
+@media (max-width: 960px) {
+    .sponsor-form {
+        grid-template-columns: repeat(auto-fit, minmax(240px, 1fr));
+    }
+
+    .sponsor-logo-group {
+        grid-column: 1 / -1;
+        grid-row: auto;
+        flex-direction: row;
+        align-items: center;
+        gap: 1.5rem;
+    }
+
+    .logo-upload {
+        width: 160px;
+    }
+
+    .span-two {
+        grid-column: 1 / -1;
+    }
+}
+
+/* Rich Text Editor */
+.editor-toolbar {
+    display: flex;
+    gap: 0.5rem;
+    padding: 0.5rem;
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    border-bottom: none;
+}
+
+.editor-btn {
+    padding: 0.25rem 0.5rem;
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    cursor: pointer;
+}
+
+.editor-btn:hover {
+    background: rgba(80, 255, 255, 0.1);
+    border-color: var(--primary-cyan);
+}
+
+.editor-content {
+    min-height: 300px;
+    padding: 1rem;
+    background: var(--bg-dark);
+    border: 1px solid var(--border-color);
+    font-family: 'Dank Mono', Monaco, monospace;
+}
+
+/* Responsive */
+@media (max-width: 1024px) {
+    .admin-layout {
+        flex-direction: column;
+    }
+
+    .admin-sidebar {
+        width: 100%;
+        border-right: none;
+        border-bottom: 1px solid var(--border-color);
+    }
+
+    .sidebar-nav {
+        display: flex;
+        overflow-x: auto;
+        padding: 0;
+    }
+
+    .nav-btn {
+        border-left: none;
+        border-bottom: 3px solid transparent;
+        white-space: nowrap;
+    }
+
+    .nav-btn.active {
+        border-bottom-color: var(--primary-cyan);
+    }
+
+    .sidebar-actions {
+        display: none;
+    }
+}

docs/md_v2/marketplace/admin/admin.js

@@ -0,0 +1,933 @@
+// Admin Dashboard - Smart & Powerful
+const { API_BASE, API_ORIGIN } = (() => {
+    const cleanOrigin = (value) => value ? value.replace(/\/$/, '') : '';
+    const params = new URLSearchParams(window.location.search);
+    const overrideParam = cleanOrigin(params.get('api_origin'));
+
+    let storedOverride = '';
+    try {
+        storedOverride = cleanOrigin(localStorage.getItem('marketplace_api_origin'));
+    } catch (error) {
+        storedOverride = '';
+    }
+
+    let origin = overrideParam || storedOverride;
+
+    if (overrideParam && overrideParam !== storedOverride) {
+        try {
+            localStorage.setItem('marketplace_api_origin', overrideParam);
+        } catch (error) {
+            // ignore storage errors (private mode, etc.)
+        }
+    }
+
+    const { protocol, hostname, port } = window.location;
+    const isLocalHost = ['localhost', '127.0.0.1', '0.0.0.0'].includes(hostname);
+
+    if (!origin && isLocalHost && port !== '8100') {
+        origin = `${protocol}//127.0.0.1:8100`;
+    }
+
+    if (origin) {
+        const normalized = cleanOrigin(origin);
+        return { API_BASE: `${normalized}/marketplace/api`, API_ORIGIN: normalized };
+    }
+
+    return { API_BASE: '/marketplace/api', API_ORIGIN: '' };
+})();
+
+const resolveAssetUrl = (path) => {
+    if (!path) return '';
+    if (/^https?:\/\//i.test(path)) return path;
+    if (path.startsWith('/') && API_ORIGIN) {
+        return `${API_ORIGIN}${path}`;
+    }
+    return path;
+};
+
+class AdminDashboard {
+    constructor() {
+        this.token = localStorage.getItem('admin_token');
+        this.currentSection = 'stats';
+        this.data = {
+            apps: [],
+            articles: [],
+            categories: [],
+            sponsors: []
+        };
+        this.editingItem = null;
+        this.init();
+    }
+
+    async init() {
+        // Check auth
+        if (!this.token) {
+            this.showLogin();
+            return;
+        }
+
+        // Try to load stats to verify token
+        try {
+            await this.loadStats();
+            this.showDashboard();
+            this.setupEventListeners();
+            await this.loadAllData();
+        } catch (error) {
+            if (error.status === 401) {
+                this.showLogin();
+            }
+        }
+    }
+
+    showLogin() {
+        document.getElementById('login-screen').classList.remove('hidden');
+        document.getElementById('admin-dashboard').classList.add('hidden');
+
+        // Set up login button click handler
+        const loginBtn = document.getElementById('login-btn');
+        if (loginBtn) {
+            loginBtn.onclick = async () => {
+                const password = document.getElementById('password').value;
+                await this.login(password);
+            };
+        }
+    }
+
+    async login(password) {
+        try {
+            const response = await fetch(`${API_BASE}/admin/login`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ password })
+            });
+
+            if (!response.ok) throw new Error('Invalid password');
+
+            const data = await response.json();
+            this.token = data.token;
+            localStorage.setItem('admin_token', this.token);
+
+            document.getElementById('login-screen').classList.add('hidden');
+            this.showDashboard();
+            this.setupEventListeners();
+            await this.loadAllData();
+        } catch (error) {
+            document.getElementById('login-error').textContent = 'Invalid password';
+            document.getElementById('password').value = '';
+        }
+    }
+
+    showDashboard() {
+        document.getElementById('login-screen').classList.add('hidden');
+        document.getElementById('admin-dashboard').classList.remove('hidden');
+    }
+
+    setupEventListeners() {
+        // Navigation
+        document.querySelectorAll('.nav-btn').forEach(btn => {
+            btn.onclick = () => this.switchSection(btn.dataset.section);
+        });
+
+        // Logout
+        document.getElementById('logout-btn').onclick = () => this.logout();
+
+        // Export/Backup
+        document.getElementById('export-btn').onclick = () => this.exportData();
+        document.getElementById('backup-btn').onclick = () => this.backupDatabase();
+
+        // Search
+        ['apps', 'articles'].forEach(type => {
+            const searchInput = document.getElementById(`${type}-search`);
+            if (searchInput) {
+                searchInput.oninput = (e) => this.filterTable(type, e.target.value);
+            }
+        });
+
+        // Category filter
+        const categoryFilter = document.getElementById('apps-filter');
+        if (categoryFilter) {
+            categoryFilter.onchange = (e) => this.filterByCategory(e.target.value);
+        }
+
+        // Save button in modal
+        document.getElementById('save-btn').onclick = () => this.saveItem();
+    }
+
+    async loadAllData() {
+        try {
+            await this.loadStats();
+        } catch (e) {
+            console.error('Failed to load stats:', e);
+        }
+
+        try {
+            await this.loadApps();
+        } catch (e) {
+            console.error('Failed to load apps:', e);
+        }
+
+        try {
+            await this.loadArticles();
+        } catch (e) {
+            console.error('Failed to load articles:', e);
+        }
+
+        try {
+            await this.loadCategories();
+        } catch (e) {
+            console.error('Failed to load categories:', e);
+        }
+
+        try {
+            await this.loadSponsors();
+        } catch (e) {
+            console.error('Failed to load sponsors:', e);
+        }
+
+        this.populateCategoryFilter();
+    }
+
+    async apiCall(endpoint, options = {}) {
+        const isFormData = options.body instanceof FormData;
+        const headers = {
+            'Authorization': `Bearer ${this.token}`,
+            ...options.headers
+        };
+
+        if (!isFormData && !headers['Content-Type']) {
+            headers['Content-Type'] = 'application/json';
+        }
+
+        const response = await fetch(`${API_BASE}${endpoint}`, {
+            ...options,
+            headers
+        });
+
+        if (response.status === 401) {
+            this.logout();
+            throw { status: 401 };
+        }
+
+        if (!response.ok) throw new Error(`API Error: ${response.status}`);
+        return response.json();
+    }
+
+    async loadStats() {
+        const stats = await this.apiCall(`/admin/stats?_=${Date.now()}`, {
+            cache: 'no-store'
+        });
+
+        document.getElementById('stat-apps').textContent = stats.apps.total;
+        document.getElementById('stat-featured').textContent = stats.apps.featured;
+        document.getElementById('stat-sponsored').textContent = stats.apps.sponsored;
+        document.getElementById('stat-articles').textContent = stats.articles;
+        document.getElementById('stat-sponsors').textContent = stats.sponsors.active;
+        document.getElementById('stat-views').textContent = this.formatNumber(stats.total_views);
+    }
+
+    async loadApps() {
+        this.data.apps = await this.apiCall(`/apps?limit=100&_=${Date.now()}`, {
+            cache: 'no-store'
+        });
+        this.renderAppsTable(this.data.apps);
+    }
+
+    async loadArticles() {
+        this.data.articles = await this.apiCall(`/articles?limit=100&_=${Date.now()}`, {
+            cache: 'no-store'
+        });
+        this.renderArticlesTable(this.data.articles);
+    }
+
+    async loadCategories() {
+        const cacheBuster = Date.now();
+        this.data.categories = await this.apiCall(`/categories?_=${cacheBuster}`, {
+            cache: 'no-store'
+        });
+        this.renderCategoriesTable(this.data.categories);
+    }
+
+    async loadSponsors() {
+        const cacheBuster = Date.now();
+        this.data.sponsors = await this.apiCall(`/sponsors?limit=100&_=${cacheBuster}`, {
+            cache: 'no-store'
+        });
+        this.renderSponsorsTable(this.data.sponsors);
+    }
+
+    renderAppsTable(apps) {
+        const table = document.getElementById('apps-table');
+        table.innerHTML = `
+            <table>
+                <thead>
+                    <tr>
+                        <th>ID</th>
+                        <th>Name</th>
+                        <th>Category</th>
+                        <th>Type</th>
+                        <th>Rating</th>
+                        <th>Downloads</th>
+                        <th>Status</th>
+                        <th>Actions</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    ${apps.map(app => `
+                        <tr>
+                            <td>${app.id}</td>
+                            <td>${app.name}</td>
+                            <td>${app.category}</td>
+                            <td>${app.type}</td>
+                            <td>◆ ${app.rating}/5</td>
+                            <td>${this.formatNumber(app.downloads)}</td>
+                            <td>
+                                ${app.featured ? '<span class="badge featured">Featured</span>' : ''}
+                                ${app.sponsored ? '<span class="badge sponsored">Sponsored</span>' : ''}
+                            </td>
+                            <td>
+                                <div class="table-actions">
+                                    <button class="btn-edit" onclick="admin.editItem('apps', ${app.id})">Edit</button>
+                                    <button class="btn-duplicate" onclick="admin.duplicateItem('apps', ${app.id})">Duplicate</button>
+                                    <button class="btn-delete" onclick="admin.deleteItem('apps', ${app.id})">Delete</button>
+                                </div>
+                            </td>
+                        </tr>
+                    `).join('')}
+                </tbody>
+            </table>
+        `;
+    }
+
+    renderArticlesTable(articles) {
+        const table = document.getElementById('articles-table');
+        table.innerHTML = `
+            <table>
+                <thead>
+                    <tr>
+                        <th>ID</th>
+                        <th>Title</th>
+                        <th>Category</th>
+                        <th>Author</th>
+                        <th>Published</th>
+                        <th>Views</th>
+                        <th>Actions</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    ${articles.map(article => `
+                        <tr>
+                            <td>${article.id}</td>
+                            <td>${article.title}</td>
+                            <td>${article.category}</td>
+                            <td>${article.author}</td>
+                            <td>${new Date(article.published_date).toLocaleDateString()}</td>
+                            <td>${this.formatNumber(article.views)}</td>
+                            <td>
+                                <div class="table-actions">
+                                    <button class="btn-edit" onclick="admin.editItem('articles', ${article.id})">Edit</button>
+                                    <button class="btn-duplicate" onclick="admin.duplicateItem('articles', ${article.id})">Duplicate</button>
+                                    <button class="btn-delete" onclick="admin.deleteItem('articles', ${article.id})">Delete</button>
+                                </div>
+                            </td>
+                        </tr>
+                    `).join('')}
+                </tbody>
+            </table>
+        `;
+    }
+
+    renderCategoriesTable(categories) {
+        const table = document.getElementById('categories-table');
+        table.innerHTML = `
+            <table>
+                <thead>
+                    <tr>
+                        <th>Order</th>
+                        <th>Icon</th>
+                        <th>Name</th>
+                        <th>Description</th>
+                        <th>Actions</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    ${categories.map(cat => `
+                        <tr>
+                            <td>${cat.order_index}</td>
+                            <td>${cat.icon}</td>
+                            <td>${cat.name}</td>
+                            <td>${cat.description}</td>
+                            <td>
+                                <div class="table-actions">
+                                    <button class="btn-edit" onclick="admin.editItem('categories', ${cat.id})">Edit</button>
+                                    <button class="btn-delete" onclick="admin.deleteCategory(${cat.id})">Delete</button>
+                                </div>
+                            </td>
+                        </tr>
+                    `).join('')}
+                </tbody>
+            </table>
+        `;
+    }
+
+    renderSponsorsTable(sponsors) {
+        const table = document.getElementById('sponsors-table');
+        table.innerHTML = `
+            <table>
+                <thead>
+                    <tr>
+                        <th>ID</th>
+                        <th>Logo</th>
+                        <th>Company</th>
+                        <th>Tier</th>
+                        <th>Start</th>
+                        <th>End</th>
+                        <th>Status</th>
+                        <th>Actions</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    ${sponsors.map(sponsor => `
+                        <tr>
+                            <td>${sponsor.id}</td>
+                            <td>${sponsor.logo_url ? `<img class="table-logo" src="${resolveAssetUrl(sponsor.logo_url)}" alt="${sponsor.company_name} logo">` : '-'}</td>
+                            <td>${sponsor.company_name}</td>
+                            <td>${sponsor.tier}</td>
+                            <td>${new Date(sponsor.start_date).toLocaleDateString()}</td>
+                            <td>${new Date(sponsor.end_date).toLocaleDateString()}</td>
+                            <td>${sponsor.active ? '<span class="badge active">Active</span>' : 'Inactive'}</td>
+                            <td>
+                                <div class="table-actions">
+                                    <button class="btn-edit" onclick="admin.editItem('sponsors', ${sponsor.id})">Edit</button>
+                                    <button class="btn-delete" onclick="admin.deleteItem('sponsors', ${sponsor.id})">Delete</button>
+                                </div>
+                            </td>
+                        </tr>
+                    `).join('')}
+                </tbody>
+            </table>
+        `;
+    }
+
+    showAddForm(type) {
+        this.editingItem = null;
+        this.showModal(type, null);
+    }
+
+    async editItem(type, id) {
+        const item = this.data[type].find(i => i.id === id);
+        if (item) {
+            this.editingItem = item;
+            this.showModal(type, item);
+        }
+    }
+
+    async duplicateItem(type, id) {
+        const item = this.data[type].find(i => i.id === id);
+        if (item) {
+            const newItem = { ...item };
+            delete newItem.id;
+            newItem.name = `${newItem.name || newItem.title} (Copy)`;
+            if (newItem.slug) newItem.slug = `${newItem.slug}-copy-${Date.now()}`;
+
+            this.editingItem = null;
+            this.showModal(type, newItem);
+        }
+    }
+
+    showModal(type, item) {
+        const modal = document.getElementById('form-modal');
+        const title = document.getElementById('modal-title');
+        const body = document.getElementById('modal-body');
+
+        title.textContent = item ? `Edit ${type.slice(0, -1)}` : `Add New ${type.slice(0, -1)}`;
+
+        if (type === 'apps') {
+            body.innerHTML = this.getAppForm(item);
+        } else if (type === 'articles') {
+            body.innerHTML = this.getArticleForm(item);
+        } else if (type === 'categories') {
+            body.innerHTML = this.getCategoryForm(item);
+        } else if (type === 'sponsors') {
+            body.innerHTML = this.getSponsorForm(item);
+        }
+
+        modal.classList.remove('hidden');
+        modal.dataset.type = type;
+
+        if (type === 'sponsors') {
+            this.setupLogoUploadHandlers();
+        }
+    }
+
+    getAppForm(app) {
+        return `
+            <div class="form-grid">
+                <div class="form-group">
+                    <label>Name *</label>
+                    <input type="text" id="form-name" value="${app?.name || ''}" required>
+                </div>
+                <div class="form-group">
+                    <label>Slug</label>
+                    <input type="text" id="form-slug" value="${app?.slug || ''}" placeholder="auto-generated">
+                </div>
+                <div class="form-group">
+                    <label>Category</label>
+                    <select id="form-category">
+                        ${this.data.categories.map(cat =>
+                            `<option value="${cat.name}" ${app?.category === cat.name ? 'selected' : ''}>${cat.name}</option>`
+                        ).join('')}
+                    </select>
+                </div>
+                <div class="form-group">
+                    <label>Type</label>
+                    <select id="form-type">
+                        <option value="Open Source" ${app?.type === 'Open Source' ? 'selected' : ''}>Open Source</option>
+                        <option value="Paid" ${app?.type === 'Paid' ? 'selected' : ''}>Paid</option>
+                        <option value="Freemium" ${app?.type === 'Freemium' ? 'selected' : ''}>Freemium</option>
+                    </select>
+                </div>
+                <div class="form-group">
+                    <label>Rating</label>
+                    <input type="number" id="form-rating" value="${app?.rating || 4.5}" min="0" max="5" step="0.1">
+                </div>
+                <div class="form-group">
+                    <label>Downloads</label>
+                    <input type="number" id="form-downloads" value="${app?.downloads || 0}">
+                </div>
+                <div class="form-group full-width">
+                    <label>Description</label>
+                    <textarea id="form-description" rows="3">${app?.description || ''}</textarea>
+                </div>
+                <div class="form-group full-width">
+                    <label>Image URL</label>
+                    <input type="text" id="form-image" value="${app?.image || ''}" placeholder="https://...">
+                </div>
+                <div class="form-group">
+                    <label>Website URL</label>
+                    <input type="text" id="form-website" value="${app?.website_url || ''}">
+                </div>
+                <div class="form-group">
+                    <label>GitHub URL</label>
+                    <input type="text" id="form-github" value="${app?.github_url || ''}">
+                </div>
+                <div class="form-group">
+                    <label>Pricing</label>
+                    <input type="text" id="form-pricing" value="${app?.pricing || 'Free'}">
+                </div>
+                <div class="form-group">
+                    <label>Contact Email</label>
+                    <input type="email" id="form-email" value="${app?.contact_email || ''}">
+                </div>
+                <div class="form-group full-width checkbox-group">
+                    <label class="checkbox-label">
+                        <input type="checkbox" id="form-featured" ${app?.featured ? 'checked' : ''}>
+                        Featured
+                    </label>
+                    <label class="checkbox-label">
+                        <input type="checkbox" id="form-sponsored" ${app?.sponsored ? 'checked' : ''}>
+                        Sponsored
+                    </label>
+                </div>
+                <div class="form-group full-width">
+                    <label>Long Description (Markdown - Overview tab)</label>
+                    <textarea id="form-long-description" rows="10" placeholder="Enter detailed description with markdown formatting...">${app?.long_description || ''}</textarea>
+                    <small>Markdown support: **bold**, *italic*, [links](url), # headers, code blocks, lists</small>
+                </div>
+                <div class="form-group full-width">
+                    <label>Integration Guide (Markdown - Integration tab)</label>
+                    <textarea id="form-integration" rows="20" placeholder="Enter integration guide with installation, examples, and code snippets using markdown...">${app?.integration_guide || ''}</textarea>
+                    <small>Single markdown field with installation, examples, and complete guide. Code blocks get auto copy buttons.</small>
+                </div>
+                <div class="form-group full-width">
+                    <label>Documentation (Markdown - Documentation tab)</label>
+                    <textarea id="form-documentation" rows="20" placeholder="Enter documentation with API reference, examples, and best practices using markdown...">${app?.documentation || ''}</textarea>
+                    <small>Full documentation with API reference, examples, best practices, etc.</small>
+                </div>
+            </div>
+        `;
+    }
+
+    getArticleForm(article) {
+        return `
+            <div class="form-grid">
+                <div class="form-group full-width">
+                    <label>Title *</label>
+                    <input type="text" id="form-title" value="${article?.title || ''}" required>
+                </div>
+                <div class="form-group">
+                    <label>Author</label>
+                    <input type="text" id="form-author" value="${article?.author || 'Crawl4AI Team'}">
+                </div>
+                <div class="form-group">
+                    <label>Category</label>
+                    <select id="form-category">
+                        <option value="News" ${article?.category === 'News' ? 'selected' : ''}>News</option>
+                        <option value="Tutorial" ${article?.category === 'Tutorial' ? 'selected' : ''}>Tutorial</option>
+                        <option value="Review" ${article?.category === 'Review' ? 'selected' : ''}>Review</option>
+                        <option value="Comparison" ${article?.category === 'Comparison' ? 'selected' : ''}>Comparison</option>
+                    </select>
+                </div>
+                <div class="form-group full-width">
+                    <label>Featured Image URL</label>
+                    <input type="text" id="form-image" value="${article?.featured_image || ''}">
+                </div>
+                <div class="form-group full-width">
+                    <label>Content</label>
+                    <textarea id="form-content" rows="20">${article?.content || ''}</textarea>
+                </div>
+            </div>
+        `;
+    }
+
+    getCategoryForm(category) {
+        return `
+            <div class="form-grid">
+                <div class="form-group">
+                    <label>Name *</label>
+                    <input type="text" id="form-name" value="${category?.name || ''}" required>
+                </div>
+                <div class="form-group">
+                    <label>Icon</label>
+                    <input type="text" id="form-icon" value="${category?.icon || '📁'}" maxlength="2">
+                </div>
+                <div class="form-group">
+                    <label>Order</label>
+                    <input type="number" id="form-order" value="${category?.order_index || 0}">
+                </div>
+                <div class="form-group full-width">
+                    <label>Description</label>
+                    <textarea id="form-description" rows="3">${category?.description || ''}</textarea>
+                </div>
+            </div>
+        `;
+    }
+
+    getSponsorForm(sponsor) {
+        const existingFile = sponsor?.logo_url ? sponsor.logo_url.split('/').pop().split('?')[0] : '';
+        return `
+            <div class="form-grid sponsor-form">
+                <div class="form-group sponsor-logo-group">
+                    <label>Logo</label>
+                    <input type="hidden" id="form-logo-url" value="${sponsor?.logo_url || ''}">
+                    <div class="logo-upload">
+                        <div class="image-preview ${sponsor?.logo_url ? '' : 'empty'}" id="form-logo-preview">
+                            ${sponsor?.logo_url ? `<img src="${resolveAssetUrl(sponsor.logo_url)}" alt="Logo preview">` : '<span>No logo uploaded</span>'}
+                        </div>
+                        <button type="button" class="upload-btn" id="form-logo-button">Upload Logo</button>
+                        <input type="file" id="form-logo-file" accept="image/png,image/jpeg,image/webp,image/svg+xml" hidden>
+                    </div>
+                    <p class="upload-hint" id="form-logo-filename">${existingFile ? `Current: ${existingFile}` : 'No file selected'}</p>
+                </div>
+                <div class="form-group span-two">
+                    <label>Company Name *</label>
+                    <input type="text" id="form-name" value="${sponsor?.company_name || ''}" required>
+                </div>
+                <div class="form-group">
+                    <label>Tier</label>
+                    <select id="form-tier">
+                        <option value="Bronze" ${sponsor?.tier === 'Bronze' ? 'selected' : ''}>Bronze</option>
+                        <option value="Silver" ${sponsor?.tier === 'Silver' ? 'selected' : ''}>Silver</option>
+                        <option value="Gold" ${sponsor?.tier === 'Gold' ? 'selected' : ''}>Gold</option>
+                    </select>
+                </div>
+                <div class="form-group">
+                    <label>Landing URL</label>
+                    <input type="text" id="form-landing" value="${sponsor?.landing_url || ''}">
+                </div>
+                <div class="form-group">
+                    <label>Banner URL</label>
+                    <input type="text" id="form-banner" value="${sponsor?.banner_url || ''}">
+                </div>
+                <div class="form-group">
+                    <label>Start Date</label>
+                    <input type="date" id="form-start" value="${sponsor?.start_date?.split('T')[0] || ''}">
+                </div>
+                <div class="form-group">
+                    <label>End Date</label>
+                    <input type="date" id="form-end" value="${sponsor?.end_date?.split('T')[0] || ''}">
+                </div>
+                <div class="form-group">
+                    <label class="checkbox-label">
+                        <input type="checkbox" id="form-active" ${sponsor?.active ? 'checked' : ''}>
+                        Active
+                    </label>
+                </div>
+            </div>
+        `;
+    }
+
+    async saveItem() {
+        const modal = document.getElementById('form-modal');
+        const type = modal.dataset.type;
+
+        try {
+            if (type === 'sponsors') {
+                const fileInput = document.getElementById('form-logo-file');
+                if (fileInput && fileInput.files && fileInput.files[0]) {
+                    const formData = new FormData();
+                    formData.append('file', fileInput.files[0]);
+                    formData.append('folder', 'sponsors');
+
+                    const uploadResponse = await this.apiCall('/admin/upload-image', {
+                        method: 'POST',
+                        body: formData
+                    });
+
+                    if (!uploadResponse.url) {
+                        throw new Error('Image upload failed');
+                    }
+
+                    document.getElementById('form-logo-url').value = uploadResponse.url;
+                }
+            }
+
+            const data = this.collectFormData(type);
+
+            if (this.editingItem) {
+                await this.apiCall(`/admin/${type}/${this.editingItem.id}`, {
+                    method: 'PUT',
+                    body: JSON.stringify(data)
+                });
+            } else {
+                await this.apiCall(`/admin/${type}`, {
+                    method: 'POST',
+                    body: JSON.stringify(data)
+                });
+            }
+
+            this.closeModal();
+            await this[`load${type.charAt(0).toUpperCase() + type.slice(1)}`]();
+            await this.loadStats();
+        } catch (error) {
+            alert('Error saving item: ' + error.message);
+        }
+    }
+
+    collectFormData(type) {
+        const data = {};
+
+        if (type === 'apps') {
+            data.name = document.getElementById('form-name').value;
+            data.slug = document.getElementById('form-slug').value || this.generateSlug(data.name);
+            data.description = document.getElementById('form-description').value;
+            data.category = document.getElementById('form-category').value;
+            data.type = document.getElementById('form-type').value;
+            const rating = parseFloat(document.getElementById('form-rating').value);
+            const downloads = parseInt(document.getElementById('form-downloads').value, 10);
+            data.rating = Number.isFinite(rating) ? rating : 0;
+            data.downloads = Number.isFinite(downloads) ? downloads : 0;
+            data.image = document.getElementById('form-image').value;
+            data.website_url = document.getElementById('form-website').value;
+            data.github_url = document.getElementById('form-github').value;
+            data.pricing = document.getElementById('form-pricing').value;
+            data.contact_email = document.getElementById('form-email').value;
+            data.featured = document.getElementById('form-featured').checked ? 1 : 0;
+            data.sponsored = document.getElementById('form-sponsored').checked ? 1 : 0;
+            data.long_description = document.getElementById('form-long-description').value;
+            data.integration_guide = document.getElementById('form-integration').value;
+            data.documentation = document.getElementById('form-documentation').value;
+        } else if (type === 'articles') {
+            data.title = document.getElementById('form-title').value;
+            data.slug = this.generateSlug(data.title);
+            data.author = document.getElementById('form-author').value;
+            data.category = document.getElementById('form-category').value;
+            data.featured_image = document.getElementById('form-image').value;
+            data.content = document.getElementById('form-content').value;
+        } else if (type === 'categories') {
+            data.name = document.getElementById('form-name').value;
+            data.slug = this.generateSlug(data.name);
+            data.icon = document.getElementById('form-icon').value;
+            data.description = document.getElementById('form-description').value;
+            const orderIndex = parseInt(document.getElementById('form-order').value, 10);
+            data.order_index = Number.isFinite(orderIndex) ? orderIndex : 0;
+        } else if (type === 'sponsors') {
+            data.company_name = document.getElementById('form-name').value;
+            data.logo_url = document.getElementById('form-logo-url').value;
+            data.tier = document.getElementById('form-tier').value;
+            data.landing_url = document.getElementById('form-landing').value;
+            data.banner_url = document.getElementById('form-banner').value;
+            data.start_date = document.getElementById('form-start').value;
+            data.end_date = document.getElementById('form-end').value;
+            data.active = document.getElementById('form-active').checked ? 1 : 0;
+        }
+
+        return data;
+    }
+
+    setupLogoUploadHandlers() {
+        const fileInput = document.getElementById('form-logo-file');
+        const preview = document.getElementById('form-logo-preview');
+        const logoUrlInput = document.getElementById('form-logo-url');
+        const trigger = document.getElementById('form-logo-button');
+        const fileNameEl = document.getElementById('form-logo-filename');
+
+        if (!fileInput || !preview || !logoUrlInput) return;
+
+        const setFileName = (text) => {
+            if (fileNameEl) {
+                fileNameEl.textContent = text;
+            }
+        };
+
+        const setEmptyState = () => {
+            preview.innerHTML = '<span>No logo uploaded</span>';
+            preview.classList.add('empty');
+            setFileName('No file selected');
+        };
+
+        const setExistingState = () => {
+            if (logoUrlInput.value) {
+                const existingFile = logoUrlInput.value.split('/').pop().split('?')[0];
+                preview.innerHTML = `<img src="${resolveAssetUrl(logoUrlInput.value)}" alt="Logo preview">`;
+                preview.classList.remove('empty');
+                setFileName(existingFile ? `Current: ${existingFile}` : 'Current logo');
+            } else {
+                setEmptyState();
+            }
+        };
+
+        setExistingState();
+
+        if (trigger) {
+            trigger.onclick = () => fileInput.click();
+        }
+
+        fileInput.addEventListener('change', (event) => {
+            const file = event.target.files && event.target.files[0];
+
+            if (!file) {
+                setExistingState();
+                return;
+            }
+
+            setFileName(file.name);
+
+            const reader = new FileReader();
+            reader.onload = () => {
+                preview.innerHTML = `<img src="${reader.result}" alt="Logo preview">`;
+                preview.classList.remove('empty');
+            };
+            reader.readAsDataURL(file);
+        });
+    }
+
+    async deleteItem(type, id) {
+        if (!confirm(`Are you sure you want to delete this ${type.slice(0, -1)}?`)) return;
+
+        try {
+            await this.apiCall(`/admin/${type}/${id}`, { method: 'DELETE' });
+            await this[`load${type.charAt(0).toUpperCase() + type.slice(1)}`]();
+            await this.loadStats();
+        } catch (error) {
+            alert('Error deleting item: ' + error.message);
+        }
+    }
+
+    async deleteCategory(id) {
+        const hasApps = this.data.apps.some(app =>
+            app.category === this.data.categories.find(c => c.id === id)?.name
+        );
+
+        if (hasApps) {
+            alert('Cannot delete category with existing apps');
+            return;
+        }
+
+        await this.deleteItem('categories', id);
+    }
+
+    closeModal() {
+        document.getElementById('form-modal').classList.add('hidden');
+        this.editingItem = null;
+    }
+
+    switchSection(section) {
+        // Update navigation
+        document.querySelectorAll('.nav-btn').forEach(btn => {
+            btn.classList.toggle('active', btn.dataset.section === section);
+        });
+
+        // Show section
+        document.querySelectorAll('.content-section').forEach(sec => {
+            sec.classList.remove('active');
+        });
+        document.getElementById(`${section}-section`).classList.add('active');
+
+        this.currentSection = section;
+    }
+
+    filterTable(type, query) {
+        const items = this.data[type].filter(item => {
+            const searchText = Object.values(item).join(' ').toLowerCase();
+            return searchText.includes(query.toLowerCase());
+        });
+
+        if (type === 'apps') {
+            this.renderAppsTable(items);
+        } else if (type === 'articles') {
+            this.renderArticlesTable(items);
+        }
+    }
+
+    filterByCategory(category) {
+        const apps = category
+            ? this.data.apps.filter(app => app.category === category)
+            : this.data.apps;
+        this.renderAppsTable(apps);
+    }
+
+    populateCategoryFilter() {
+        const filter = document.getElementById('apps-filter');
+        if (!filter) return;
+
+        filter.innerHTML = '<option value="">All Categories</option>';
+        this.data.categories.forEach(cat => {
+            filter.innerHTML += `<option value="${cat.name}">${cat.name}</option>`;
+        });
+    }
+
+    async exportData() {
+        const data = {
+            apps: this.data.apps,
+            articles: this.data.articles,
+            categories: this.data.categories,
+            sponsors: this.data.sponsors,
+            exported: new Date().toISOString()
+        };
+
+        const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' });
+        const url = URL.createObjectURL(blob);
+        const a = document.createElement('a');
+        a.href = url;
+        a.download = `marketplace-export-${Date.now()}.json`;
+        a.click();
+    }
+
+    async backupDatabase() {
+        // In production, this would download the SQLite file
+        alert('Database backup would be implemented on the server side');
+    }
+
+    generateSlug(text) {
+        return text.toLowerCase()
+            .replace(/[^\w\s-]/g, '')
+            .replace(/\s+/g, '-')
+            .replace(/-+/g, '-')
+            .trim();
+    }
+
+    formatNumber(num) {
+        if (num >= 1000000) return (num / 1000000).toFixed(1) + 'M';
+        if (num >= 1000) return (num / 1000).toFixed(1) + 'K';
+        return num.toString();
+    }
+
+    logout() {
+        localStorage.removeItem('admin_token');
+        this.token = null;
+        this.showLogin();
+    }
+}
+
+// Initialize
+const admin = new AdminDashboard();

docs/md_v2/marketplace/admin/index.html

@@ -0,0 +1,215 @@
+<!DOCTYPE html>
+<html lang="en" data-theme="dark">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Admin Dashboard - Crawl4AI Marketplace</title>
+    <link rel="stylesheet" href="../frontend/marketplace.css?v=1759329000">
+    <link rel="stylesheet" href="admin.css?v=1759329000">
+</head>
+<body>
+    <div class="admin-container">
+        <!-- Login Screen -->
+        <div id="login-screen" class="login-screen">
+            <div class="login-box">
+                <img src="../../assets/images/logo.png" alt="Crawl4AI" class="login-logo">
+                <h1>[ Admin Access ]</h1>
+                <div id="login-form">
+                    <input type="password" id="password" placeholder="Enter admin password" autofocus onkeypress="if(event.key==='Enter'){document.getElementById('login-btn').click()}">
+                    <button type="button" id="login-btn">→ Login</button>
+                </div>
+                <div id="login-error" class="error-msg"></div>
+            </div>
+        </div>
+
+        <!-- Admin Dashboard -->
+        <div id="admin-dashboard" class="admin-dashboard hidden">
+            <!-- Header -->
+            <header class="admin-header">
+                <div class="header-content">
+                    <div class="header-left">
+                        <img src="../../assets/images/logo.png" alt="Crawl4AI" class="header-logo">
+                        <h1>[ Admin Dashboard ]</h1>
+                    </div>
+                    <div class="header-right">
+                        <span class="admin-user">Administrator</span>
+                        <button id="logout-btn" class="logout-btn">↗ Logout</button>
+                    </div>
+                </div>
+            </header>
+
+            <!-- Main Layout -->
+            <div class="admin-layout">
+                <!-- Sidebar -->
+                <aside class="admin-sidebar">
+                    <nav class="sidebar-nav">
+                        <button class="nav-btn active" data-section="stats">
+                            <span class="nav-icon">▓</span> Dashboard
+                        </button>
+                        <button class="nav-btn" data-section="apps">
+                            <span class="nav-icon">◆</span> Apps
+                        </button>
+                        <button class="nav-btn" data-section="articles">
+                            <span class="nav-icon">■</span> Articles
+                        </button>
+                        <button class="nav-btn" data-section="categories">
+                            <span class="nav-icon">□</span> Categories
+                        </button>
+                        <button class="nav-btn" data-section="sponsors">
+                            <span class="nav-icon">◆</span> Sponsors
+                        </button>
+                    </nav>
+
+                    <div class="sidebar-actions">
+                        <button id="export-btn" class="action-btn">
+                            <span>↓</span> Export Data
+                        </button>
+                        <button id="backup-btn" class="action-btn">
+                            <span>▪</span> Backup DB
+                        </button>
+                    </div>
+                </aside>
+
+                <!-- Main Content -->
+                <main class="admin-main">
+                    <!-- Stats Section -->
+                    <section id="stats-section" class="content-section active">
+                        <h2>Dashboard Overview</h2>
+                        <div class="stats-grid">
+                            <div class="stat-card">
+                                <div class="stat-icon">◆</div>
+                                <div class="stat-info">
+                                    <div class="stat-number" id="stat-apps">--</div>
+                                    <div class="stat-label">Total Apps</div>
+                                    <div class="stat-detail">
+                                        <span id="stat-featured">--</span> featured,
+                                        <span id="stat-sponsored">--</span> sponsored
+                                    </div>
+                                </div>
+                            </div>
+                            <div class="stat-card">
+                                <div class="stat-icon">■</div>
+                                <div class="stat-info">
+                                    <div class="stat-number" id="stat-articles">--</div>
+                                    <div class="stat-label">Articles</div>
+                                </div>
+                            </div>
+                            <div class="stat-card">
+                                <div class="stat-icon">◆</div>
+                                <div class="stat-info">
+                                    <div class="stat-number" id="stat-sponsors">--</div>
+                                    <div class="stat-label">Active Sponsors</div>
+                                </div>
+                            </div>
+                            <div class="stat-card">
+                                <div class="stat-icon">●</div>
+                                <div class="stat-info">
+                                    <div class="stat-number" id="stat-views">--</div>
+                                    <div class="stat-label">Total Views</div>
+                                </div>
+                            </div>
+                        </div>
+
+                        <h3>Quick Actions</h3>
+                        <div class="quick-actions">
+                            <button class="quick-btn" onclick="admin.showAddForm('apps')">
+                                <span>→</span> Add New App
+                            </button>
+                            <button class="quick-btn" onclick="admin.showAddForm('articles')">
+                                <span>→</span> Write Article
+                            </button>
+                            <button class="quick-btn" onclick="admin.showAddForm('sponsors')">
+                                <span>→</span> Add Sponsor
+                            </button>
+                        </div>
+                    </section>
+
+                    <!-- Apps Section -->
+                    <section id="apps-section" class="content-section">
+                        <div class="section-header">
+                            <h2>Apps Management</h2>
+                            <div class="header-actions">
+                                <input type="text" id="apps-search" class="search-input" placeholder="Search apps...">
+                                <select id="apps-filter" class="filter-select">
+                                    <option value="">All Categories</option>
+                                </select>
+                                <button class="add-btn" onclick="admin.showAddForm('apps')">
+                                    <span>→</span> Add App
+                                </button>
+                            </div>
+                        </div>
+                        <div class="data-table" id="apps-table">
+                            <!-- Apps table will be populated here -->
+                        </div>
+                    </section>
+
+                    <!-- Articles Section -->
+                    <section id="articles-section" class="content-section">
+                        <div class="section-header">
+                            <h2>Articles Management</h2>
+                            <div class="header-actions">
+                                <input type="text" id="articles-search" class="search-input" placeholder="Search articles...">
+                                <button class="add-btn" onclick="admin.showAddForm('articles')">
+                                    <span>→</span> Add Article
+                                </button>
+                            </div>
+                        </div>
+                        <div class="data-table" id="articles-table">
+                            <!-- Articles table will be populated here -->
+                        </div>
+                    </section>
+
+                    <!-- Categories Section -->
+                    <section id="categories-section" class="content-section">
+                        <div class="section-header">
+                            <h2>Categories Management</h2>
+                            <div class="header-actions">
+                                <button class="add-btn" onclick="admin.showAddForm('categories')">
+                                    <span>→</span> Add Category
+                                </button>
+                            </div>
+                        </div>
+                        <div class="data-table" id="categories-table">
+                            <!-- Categories table will be populated here -->
+                        </div>
+                    </section>
+
+                    <!-- Sponsors Section -->
+                    <section id="sponsors-section" class="content-section">
+                        <div class="section-header">
+                            <h2>Sponsors Management</h2>
+                            <div class="header-actions">
+                                <button class="add-btn" onclick="admin.showAddForm('sponsors')">
+                                    <span>→</span> Add Sponsor
+                                </button>
+                            </div>
+                        </div>
+                        <div class="data-table" id="sponsors-table">
+                            <!-- Sponsors table will be populated here -->
+                        </div>
+                    </section>
+                </main>
+            </div>
+        </div>
+
+        <!-- Modal for Add/Edit Forms -->
+        <div id="form-modal" class="modal hidden">
+            <div class="modal-content large">
+                <div class="modal-header">
+                    <h2 id="modal-title">Add/Edit</h2>
+                    <button class="modal-close" onclick="admin.closeModal()">✕</button>
+                </div>
+                <div class="modal-body" id="modal-body">
+                    <!-- Dynamic form content -->
+                </div>
+                <div class="modal-footer">
+                    <button class="btn-cancel" onclick="admin.closeModal()">Cancel</button>
+                    <button class="btn-save" id="save-btn">Save</button>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <script src="admin.js?v=1759335000"></script>
+</body>
+</html>

docs/md_v2/marketplace/app-detail.css

@@ -0,0 +1,683 @@
+/* App Detail Page Styles */
+
+.app-detail-container {
+    min-height: 100vh;
+    background: var(--bg-dark);
+}
+
+/* Back Button */
+.header-nav {
+    display: flex;
+    align-items: center;
+}
+
+.back-btn {
+    padding: 0.5rem 1rem;
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--primary-cyan);
+    text-decoration: none;
+    transition: all 0.2s;
+    font-size: 0.875rem;
+}
+
+.back-btn:hover {
+    border-color: var(--primary-cyan);
+    background: rgba(80, 255, 255, 0.1);
+}
+
+/* App Hero Section */
+.app-hero {
+    max-width: 1800px;
+    margin: 2rem auto;
+    padding: 0 2rem;
+}
+
+.app-hero-content {
+    display: grid;
+    grid-template-columns: 1fr 2fr;
+    gap: 3rem;
+    background: linear-gradient(135deg, #1a1a2e, #0f0f1e);
+    border: 2px solid var(--primary-cyan);
+    padding: 2rem;
+    box-shadow: 0 0 30px rgba(80, 255, 255, 0.15),
+                inset 0 0 20px rgba(80, 255, 255, 0.05);
+}
+
+.app-hero-image {
+    width: 100%;
+    height: 300px;
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.1), rgba(243, 128, 245, 0.05));
+    background-size: cover;
+    background-position: center;
+    border: 1px solid var(--border-color);
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-size: 4rem;
+    color: var(--primary-cyan);
+}
+
+.app-badges {
+    display: flex;
+    gap: 0.5rem;
+    margin-bottom: 1rem;
+}
+
+.app-badge {
+    padding: 0.3rem 0.6rem;
+    background: var(--bg-tertiary);
+    color: var(--text-secondary);
+    font-size: 0.75rem;
+    text-transform: uppercase;
+    font-weight: 600;
+}
+
+.app-badge.featured {
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    color: var(--bg-dark);
+    box-shadow: 0 2px 10px rgba(80, 255, 255, 0.3);
+}
+
+.app-badge.sponsored {
+    background: linear-gradient(135deg, var(--warning), #ff8c00);
+    color: var(--bg-dark);
+    box-shadow: 0 2px 10px rgba(245, 158, 11, 0.3);
+}
+
+.app-hero-info h1 {
+    font-size: 2.5rem;
+    color: var(--primary-cyan);
+    margin: 0.5rem 0;
+    text-shadow: 0 0 20px rgba(80, 255, 255, 0.5);
+}
+
+.app-tagline {
+    font-size: 1.1rem;
+    color: var(--text-secondary);
+    margin-bottom: 2rem;
+}
+
+/* Stats */
+.app-stats {
+    display: flex;
+    gap: 2rem;
+    margin: 2rem 0;
+    padding: 1rem 0;
+    border-top: 1px solid var(--border-color);
+    border-bottom: 1px solid var(--border-color);
+}
+
+.stat {
+    display: flex;
+    flex-direction: column;
+    gap: 0.25rem;
+}
+
+.stat-value {
+    font-size: 1.5rem;
+    color: var(--primary-cyan);
+    font-weight: 600;
+}
+
+.stat-label {
+    font-size: 0.875rem;
+    color: var(--text-tertiary);
+}
+
+/* Action Buttons */
+.app-actions {
+    display: flex;
+    gap: 1rem;
+    margin: 2rem 0;
+}
+
+.action-btn {
+    padding: 0.75rem 1.5rem;
+    border: 1px solid var(--border-color);
+    background: transparent;
+    color: var(--text-primary);
+    text-decoration: none;
+    display: inline-flex;
+    align-items: center;
+    gap: 0.5rem;
+    transition: all 0.2s;
+    cursor: pointer;
+    font-family: inherit;
+    font-size: 0.9rem;
+}
+
+.action-btn.primary {
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    color: var(--bg-dark);
+    border-color: var(--primary-cyan);
+    font-weight: 600;
+}
+
+.action-btn.primary:hover {
+    box-shadow: 0 4px 15px rgba(80, 255, 255, 0.3);
+    transform: translateY(-2px);
+}
+
+.action-btn.secondary {
+    border-color: var(--accent-pink);
+    color: var(--accent-pink);
+}
+
+.action-btn.secondary:hover {
+    background: rgba(243, 128, 245, 0.1);
+    box-shadow: 0 4px 15px rgba(243, 128, 245, 0.2);
+}
+
+.action-btn.ghost {
+    border-color: var(--border-color);
+    color: var(--text-secondary);
+}
+
+.action-btn.ghost:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+/* Pricing */
+.pricing-info {
+    display: flex;
+    align-items: center;
+    gap: 1rem;
+    font-size: 1.1rem;
+}
+
+.pricing-label {
+    color: var(--text-tertiary);
+}
+
+.pricing-value {
+    color: var(--warning);
+    font-weight: 600;
+}
+
+/* Navigation Tabs */
+.tabs {
+    display: flex;
+    flex-direction: row;
+    gap: 0;
+    border-bottom: 2px solid var(--border-color);
+    margin-bottom: 0;
+    background: var(--bg-tertiary);
+}
+
+.tab-btn {
+    padding: 1rem 2rem;
+    background: transparent;
+    border: none;
+    border-bottom: 3px solid transparent;
+    color: var(--text-secondary);
+    cursor: pointer;
+    transition: all 0.2s;
+    font-family: inherit;
+    font-size: 0.95rem;
+    margin-bottom: -2px;
+    white-space: nowrap;
+    font-weight: 500;
+}
+
+.tab-btn:hover {
+    color: var(--primary-cyan);
+    background: rgba(80, 255, 255, 0.05);
+}
+
+.tab-btn.active {
+    color: var(--primary-cyan);
+    border-bottom-color: var(--primary-cyan);
+    background: var(--bg-secondary);
+}
+
+.app-nav {
+    max-width: 1800px;
+    margin: 2rem auto 0;
+    padding: 0 2rem;
+    display: flex;
+    gap: 1rem;
+    border-bottom: 2px solid var(--border-color);
+}
+
+.nav-tab {
+    padding: 1rem 1.5rem;
+    background: transparent;
+    border: none;
+    border-bottom: 2px solid transparent;
+    color: var(--text-secondary);
+    cursor: pointer;
+    transition: all 0.2s;
+    font-family: inherit;
+    font-size: 0.9rem;
+    margin-bottom: -2px;
+}
+
+.nav-tab:hover {
+    color: var(--primary-cyan);
+}
+
+.nav-tab.active {
+    color: var(--primary-cyan);
+    border-bottom-color: var(--primary-cyan);
+}
+
+/* Main Content Wrapper */
+.app-main {
+    max-width: 1800px;
+    margin: 2rem auto;
+    padding: 0 2rem;
+}
+
+/* Content Sections */
+.app-content {
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    padding: 0;
+}
+
+.tab-content {
+    display: none;
+    padding: 2rem;
+}
+
+.tab-content.active {
+    display: block;
+}
+
+/* Overview Layout */
+.overview-columns {
+    display: grid;
+    grid-template-columns: 2fr 1fr;
+    gap: 2rem;
+}
+
+.overview-main h2, .overview-main h3 {
+    color: var(--primary-cyan);
+    margin-top: 2rem;
+    margin-bottom: 1rem;
+}
+
+.overview-main h2:first-child {
+    margin-top: 0;
+}
+
+.overview-main h2 {
+    font-size: 1.8rem;
+    border-bottom: 2px solid var(--border-color);
+    padding-bottom: 0.5rem;
+}
+
+.overview-main h3 {
+    font-size: 1.3rem;
+}
+
+.features-list {
+    list-style: none;
+    padding: 0;
+}
+
+.features-list li {
+    padding: 0.5rem 0;
+    padding-left: 1.5rem;
+    position: relative;
+    color: var(--text-secondary);
+}
+
+.features-list li:before {
+    content: "▸";
+    position: absolute;
+    left: 0;
+    color: var(--primary-cyan);
+}
+
+.use-cases p {
+    color: var(--text-secondary);
+    line-height: 1.6;
+}
+
+/* Sidebar */
+.sidebar {
+    display: flex;
+    flex-direction: column;
+    gap: 1rem;
+}
+
+.sidebar-card {
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    padding: 1.5rem;
+}
+
+.sidebar-card h3 {
+    font-size: 1.1rem;
+    color: var(--primary-cyan);
+    margin: 0 0 1rem 0;
+    border-bottom: 1px solid var(--border-color);
+    padding-bottom: 0.5rem;
+}
+
+.stats-grid {
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    gap: 1rem;
+}
+
+.stats-grid > div {
+    text-align: center;
+}
+
+.metadata {
+    margin: 0;
+}
+
+.metadata div {
+    display: flex;
+    justify-content: space-between;
+    padding: 0.75rem 0;
+    border-bottom: 1px solid var(--border-color);
+}
+
+.metadata dt {
+    color: var(--text-tertiary);
+    font-weight: normal;
+}
+
+.metadata dd {
+    color: var(--text-primary);
+    margin: 0;
+    font-weight: 600;
+}
+
+.sidebar-card p {
+    color: var(--text-secondary);
+    margin: 0;
+}
+
+/* Integration Content */
+.integration-content {
+    max-width: 100%;
+}
+
+.integration-content h2 {
+    font-size: 1.8rem;
+    color: var(--primary-cyan);
+    margin: 0 0 2rem 0;
+    padding-bottom: 0.5rem;
+    border-bottom: 2px solid var(--border-color);
+}
+
+.integration-content h3 {
+    font-size: 1.3rem;
+    color: var(--text-primary);
+    margin: 2rem 0 1rem;
+}
+
+.docs-content {
+    max-width: 100%;
+}
+
+.docs-content h2 {
+    font-size: 1.8rem;
+    color: var(--primary-cyan);
+    margin: 0 0 1.5rem 0;
+    padding-bottom: 0.5rem;
+    border-bottom: 2px solid var(--border-color);
+}
+
+.docs-content h3 {
+    font-size: 1.3rem;
+    color: var(--text-primary);
+    margin: 2rem 0 1rem;
+}
+
+.docs-content h4 {
+    font-size: 1.1rem;
+    color: var(--accent-pink);
+    margin: 1.5rem 0 0.5rem;
+}
+
+.docs-content p {
+    color: var(--text-secondary);
+    line-height: 1.6;
+    margin-bottom: 1rem;
+}
+
+.docs-content code {
+    background: var(--bg-tertiary);
+    padding: 0.2rem 0.4rem;
+    color: var(--primary-cyan);
+    font-family: 'Dank Mono', Monaco, monospace;
+    font-size: 0.9em;
+}
+
+/* Code Blocks */
+.code-block {
+    background: var(--bg-dark);
+    border: 1px solid var(--border-color);
+    margin: 1rem 0;
+    overflow: hidden;
+    position: relative;
+}
+
+.code-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    padding: 0.5rem 1rem;
+    background: var(--bg-tertiary);
+    border-bottom: 1px solid var(--border-color);
+}
+
+.code-lang {
+    color: var(--primary-cyan);
+    font-size: 0.875rem;
+    text-transform: uppercase;
+}
+
+.copy-btn {
+    position: absolute;
+    top: 0.5rem;
+    right: 0.5rem;
+    padding: 0.4rem 0.8rem;
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    cursor: pointer;
+    font-size: 0.75rem;
+    transition: all 0.2s;
+    z-index: 10;
+}
+
+.copy-btn:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+    background: var(--bg-secondary);
+}
+
+.code-block pre {
+    margin: 0;
+    padding: 1rem;
+    overflow-x: auto;
+}
+
+.code-block code {
+    background: transparent;
+    padding: 0;
+    color: var(--text-secondary);
+    font-size: 0.875rem;
+    line-height: 1.5;
+}
+
+/* Markdown rendered code blocks */
+.integration-content pre,
+.docs-content pre {
+    background: var(--bg-dark);
+    border: 1px solid var(--border-color);
+    margin: 1rem 0;
+    padding: 1rem;
+    padding-top: 2.5rem; /* Space for copy button */
+    overflow-x: auto;
+    position: relative;
+    max-height: none; /* Remove any height restrictions */
+    height: auto; /* Allow content to expand */
+}
+
+.integration-content pre code,
+.docs-content pre code {
+    background: transparent;
+    padding: 0;
+    color: var(--text-secondary);
+    font-size: 0.875rem;
+    line-height: 1.5;
+    white-space: pre; /* Preserve whitespace and line breaks */
+    display: block;
+}
+
+/* Feature Grid */
+.feature-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+    gap: 1rem;
+    margin: 2rem 0;
+}
+
+.feature-card {
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    padding: 1.5rem;
+    transition: all 0.2s;
+}
+
+.feature-card:hover {
+    border-color: var(--primary-cyan);
+    background: rgba(80, 255, 255, 0.05);
+}
+
+.feature-card h4 {
+    margin-top: 0;
+}
+
+/* Info Box */
+.info-box {
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.05), rgba(243, 128, 245, 0.03));
+    border: 1px solid var(--primary-cyan);
+    border-left: 4px solid var(--primary-cyan);
+    padding: 1.5rem;
+    margin: 2rem 0;
+}
+
+.info-box h4 {
+    margin-top: 0;
+    color: var(--primary-cyan);
+}
+
+/* Support Grid */
+.support-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+    gap: 1rem;
+    margin: 2rem 0;
+}
+
+.support-card {
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    padding: 1.5rem;
+    text-align: center;
+}
+
+.support-card h3 {
+    color: var(--primary-cyan);
+    margin-bottom: 0.5rem;
+}
+
+/* Related Apps */
+.related-apps {
+    max-width: 1800px;
+    margin: 4rem auto;
+    padding: 0 2rem;
+}
+
+.related-apps h2 {
+    font-size: 1.5rem;
+    color: var(--text-primary);
+    margin-bottom: 1.5rem;
+}
+
+.related-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
+    gap: 1rem;
+}
+
+.related-app-card {
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    padding: 1rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.related-app-card:hover {
+    border-color: var(--primary-cyan);
+    transform: translateY(-2px);
+}
+
+/* Responsive */
+@media (max-width: 1024px) {
+    .app-hero-content {
+        grid-template-columns: 1fr;
+    }
+
+    .app-stats {
+        justify-content: space-around;
+    }
+
+    .overview-columns {
+        grid-template-columns: 1fr;
+    }
+}
+
+@media (max-width: 768px) {
+    .app-hero-info h1 {
+        font-size: 2rem;
+    }
+
+    .app-actions {
+        flex-direction: column;
+    }
+
+    .tabs {
+        overflow-x: auto;
+        -webkit-overflow-scrolling: touch;
+    }
+
+    .tab-btn {
+        padding: 0.75rem 1.5rem;
+        font-size: 0.875rem;
+    }
+
+    .app-nav {
+        overflow-x: auto;
+        gap: 0;
+    }
+
+    .nav-tab {
+        white-space: nowrap;
+    }
+
+    .feature-grid,
+    .support-grid {
+        grid-template-columns: 1fr;
+    }
+
+    .tab-content {
+        padding: 1rem;
+    }
+
+    .app-main {
+        padding: 0 1rem;
+    }
+}

docs/md_v2/marketplace/app-detail.html

@@ -0,0 +1,177 @@
+<!DOCTYPE html>
+<html lang="en" data-theme="dark">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>App Details - Crawl4AI Marketplace</title>
+    <link rel="stylesheet" href="marketplace.css">
+    <link rel="stylesheet" href="app-detail.css">
+</head>
+<body>
+    <div class="app-detail-container">
+        <!-- Header -->
+        <header class="marketplace-header">
+            <div class="header-content">
+                <div class="header-left">
+                    <div class="logo-title">
+                        <img src="../assets/images/logo.png" alt="Crawl4AI" class="header-logo">
+                        <h1>
+                            <span class="ascii-border">[</span>
+                            Marketplace
+                            <span class="ascii-border">]</span>
+                        </h1>
+                    </div>
+                </div>
+                <div class="header-nav">
+                    <a href="index.html" class="back-btn">← Back to Marketplace</a>
+                </div>
+            </div>
+        </header>
+
+        <!-- App Hero Section -->
+        <section class="app-hero">
+            <div class="app-hero-content">
+                <div class="app-hero-image" id="app-image">
+                    <!-- Dynamic image -->
+                </div>
+                <div class="app-hero-info">
+                    <div class="app-badges">
+                        <span class="app-badge" id="app-type">Open Source</span>
+                        <span class="app-badge featured" id="app-featured" style="display:none">FEATURED</span>
+                        <span class="app-badge sponsored" id="app-sponsored" style="display:none">SPONSORED</span>
+                    </div>
+                    <h1 id="app-name">App Name</h1>
+                    <p id="app-description" class="app-tagline">App description goes here</p>
+
+                    <div class="app-stats">
+                        <div class="stat">
+                            <span class="stat-value" id="app-rating">★★★★★</span>
+                            <span class="stat-label">Rating</span>
+                        </div>
+                        <div class="stat">
+                            <span class="stat-value" id="app-downloads">0</span>
+                            <span class="stat-label">Downloads</span>
+                        </div>
+                        <div class="stat">
+                            <span class="stat-value" id="app-category">Category</span>
+                            <span class="stat-label">Category</span>
+                        </div>
+                    </div>
+
+                    <div class="app-actions">
+                        <a href="#" id="app-website" class="action-btn primary" target="_blank">Visit Website</a>
+                        <a href="#" id="app-github" class="action-btn" target="_blank">View GitHub</a>
+                        <a href="#" id="app-demo" class="action-btn" target="_blank" style="display:none">Live Demo</a>
+                    </div>
+                </div>
+            </div>
+        </section>
+
+        <!-- App Details Section -->
+        <main class="app-main">
+            <div class="app-content">
+                <div class="tabs">
+                    <button class="tab-btn active" data-tab="overview">Overview</button>
+                    <button class="tab-btn" data-tab="integration">Integration</button>
+                    <button class="tab-btn" data-tab="docs">Documentation</button>
+                    <button class="tab-btn" data-tab="support">Support</button>
+                </div>
+
+                <section id="overview-tab" class="tab-content active">
+                    <div class="overview-columns">
+                        <div class="overview-main">
+                            <div id="app-overview">Overview content goes here.</div>
+                        </div>
+
+                        <aside class="sidebar">
+                            <div class="sidebar-card">
+                                <h3>Download Stats</h3>
+                                <div class="stats-grid">
+                                    <div>
+                                        <span class="stat-value" id="sidebar-downloads">0</span>
+                                        <span class="stat-label">Downloads</span>
+                                    </div>
+                                    <div>
+                                        <span class="stat-value" id="sidebar-rating">0.0</span>
+                                        <span class="stat-label">Rating</span>
+                                    </div>
+                                </div>
+                            </div>
+
+                            <div class="sidebar-card">
+                                <h3>App Metadata</h3>
+                                <dl class="metadata">
+                                    <div>
+                                        <dt>Category</dt>
+                                        <dd id="sidebar-category">-</dd>
+                                    </div>
+                                    <div>
+                                        <dt>Type</dt>
+                                        <dd id="sidebar-type">-</dd>
+                                    </div>
+                                    <div>
+                                        <dt>Status</dt>
+                                        <dd id="sidebar-status">Active</dd>
+                                    </div>
+                                    <div>
+                                        <dt>Pricing</dt>
+                                        <dd id="sidebar-pricing">-</dd>
+                                    </div>
+                                </dl>
+                            </div>
+
+                            <div class="sidebar-card">
+                                <h3>Contact</h3>
+                                <p id="sidebar-contact">contact@example.com</p>
+                            </div>
+                        </aside>
+                    </div>
+                </section>
+
+                <section id="integration-tab" class="tab-content">
+                    <div class="integration-content" id="app-integration">
+                        <!-- Integration guide markdown content will be rendered here -->
+                    </div>
+                </section>
+
+                <section id="docs-tab" class="tab-content">
+                    <div class="docs-content" id="app-docs">
+                        <!-- Documentation markdown content will be rendered here -->
+                    </div>
+                </section>
+
+                <section id="support-tab" class="tab-content">
+                    <div class="docs-content">
+                        <h2>Support</h2>
+                        <div class="support-grid">
+                            <div class="support-card">
+                                <h3>📧 Contact</h3>
+                                <p id="app-contact">contact@example.com</p>
+                            </div>
+                            <div class="support-card">
+                                <h3>🐛 Report Issues</h3>
+                                <p>Found a bug? Report it on GitHub Issues.</p>
+                            </div>
+                            <div class="support-card">
+                                <h3>💬 Community</h3>
+                                <p>Join our Discord for help and discussions.</p>
+                            </div>
+                        </div>
+                    </div>
+                </section>
+            </div>
+
+        </main>
+
+        <!-- Related Apps -->
+        <section class="related-apps">
+            <h2>Related Apps</h2>
+            <div id="related-apps-grid" class="related-grid">
+                <!-- Dynamic related apps -->
+            </div>
+        </section>
+    </div>
+
+    <script src="app-detail.js"></script>
+</body>
+</html>

docs/md_v2/marketplace/app-detail.js

@@ -0,0 +1,309 @@
+// App Detail Page JavaScript
+const { API_BASE, API_ORIGIN } = (() => {
+    const { hostname, port, protocol } = window.location;
+    const isLocalHost = ['localhost', '127.0.0.1', '0.0.0.0'].includes(hostname);
+
+    if (isLocalHost && port && port !== '8100') {
+        const origin = `${protocol}//127.0.0.1:8100`;
+        return { API_BASE: `${origin}/marketplace/api`, API_ORIGIN: origin };
+    }
+
+    return { API_BASE: '/marketplace/api', API_ORIGIN: '' };
+})();
+
+class AppDetailPage {
+    constructor() {
+        this.appSlug = this.getAppSlugFromURL();
+        this.appData = null;
+        this.init();
+    }
+
+    getAppSlugFromURL() {
+        const params = new URLSearchParams(window.location.search);
+        return params.get('app') || '';
+    }
+
+    async init() {
+        if (!this.appSlug) {
+            window.location.href = 'index.html';
+            return;
+        }
+
+        await this.loadAppDetails();
+        this.setupEventListeners();
+        await this.loadRelatedApps();
+    }
+
+    async loadAppDetails() {
+        try {
+            const response = await fetch(`${API_BASE}/apps/${this.appSlug}`);
+            if (!response.ok) throw new Error('App not found');
+
+            this.appData = await response.json();
+            this.renderAppDetails();
+        } catch (error) {
+            console.error('Error loading app details:', error);
+            // Fallback to loading all apps and finding the right one
+            try {
+                const response = await fetch(`${API_BASE}/apps`);
+                const apps = await response.json();
+                this.appData = apps.find(app => app.slug === this.appSlug || app.name.toLowerCase().replace(/\s+/g, '-') === this.appSlug);
+                if (this.appData) {
+                    this.renderAppDetails();
+                } else {
+                    window.location.href = 'index.html';
+                }
+            } catch (err) {
+                console.error('Error loading apps:', err);
+                window.location.href = 'index.html';
+            }
+        }
+    }
+
+    renderAppDetails() {
+        if (!this.appData) return;
+
+        // Update title
+        document.title = `${this.appData.name} - Crawl4AI Marketplace`;
+
+        // Hero image
+        const appImage = document.getElementById('app-image');
+        if (this.appData.image) {
+            appImage.style.backgroundImage = `url('${this.appData.image}')`;
+            appImage.innerHTML = '';
+        } else {
+            appImage.innerHTML = `[${this.appData.category || 'APP'}]`;
+        }
+
+        // Basic info
+        document.getElementById('app-name').textContent = this.appData.name;
+        document.getElementById('app-description').textContent = this.appData.description;
+        document.getElementById('app-type').textContent = this.appData.type || 'Open Source';
+        document.getElementById('app-category').textContent = this.appData.category;
+
+        // Badges
+        if (this.appData.featured) {
+            document.getElementById('app-featured').style.display = 'inline-block';
+        }
+        if (this.appData.sponsored) {
+            document.getElementById('app-sponsored').style.display = 'inline-block';
+        }
+
+        // Stats
+        const rating = this.appData.rating || 0;
+        const stars = '★'.repeat(Math.floor(rating)) + '☆'.repeat(5 - Math.floor(rating));
+        document.getElementById('app-rating').textContent = stars + ` ${rating}/5`;
+        document.getElementById('app-downloads').textContent = this.formatNumber(this.appData.downloads || 0);
+
+        // Action buttons
+        const websiteBtn = document.getElementById('app-website');
+        const githubBtn = document.getElementById('app-github');
+
+        if (this.appData.website_url) {
+            websiteBtn.href = this.appData.website_url;
+        } else {
+            websiteBtn.style.display = 'none';
+        }
+
+        if (this.appData.github_url) {
+            githubBtn.href = this.appData.github_url;
+        } else {
+            githubBtn.style.display = 'none';
+        }
+
+        // Contact
+        document.getElementById('app-contact').textContent = this.appData.contact_email || 'Not available';
+
+        // Sidebar info
+        document.getElementById('sidebar-downloads').textContent = this.formatNumber(this.appData.downloads || 0);
+        document.getElementById('sidebar-rating').textContent = (this.appData.rating || 0).toFixed(1);
+        document.getElementById('sidebar-category').textContent = this.appData.category || '-';
+        document.getElementById('sidebar-type').textContent = this.appData.type || '-';
+        document.getElementById('sidebar-status').textContent = this.appData.status || 'Active';
+        document.getElementById('sidebar-pricing').textContent = this.appData.pricing || 'Free';
+        document.getElementById('sidebar-contact').textContent = this.appData.contact_email || 'contact@example.com';
+
+        // Render tab contents from database fields
+        this.renderTabContents();
+    }
+
+    renderTabContents() {
+        // Overview tab - use long_description from database
+        const overviewDiv = document.getElementById('app-overview');
+        if (overviewDiv) {
+            if (this.appData.long_description) {
+                overviewDiv.innerHTML = this.renderMarkdown(this.appData.long_description);
+            } else {
+                overviewDiv.innerHTML = `<p>${this.appData.description || 'No overview available.'}</p>`;
+            }
+        }
+
+        // Integration tab - use integration_guide field from database
+        const integrationDiv = document.getElementById('app-integration');
+        if (integrationDiv) {
+            if (this.appData.integration_guide) {
+                integrationDiv.innerHTML = this.renderMarkdown(this.appData.integration_guide);
+                // Add copy buttons to all code blocks
+                this.addCopyButtonsToCodeBlocks(integrationDiv);
+            } else {
+                integrationDiv.innerHTML = '<p>Integration guide not yet available. Please check the official website for details.</p>';
+            }
+        }
+
+        // Documentation tab - use documentation field from database
+        const docsDiv = document.getElementById('app-docs');
+        if (docsDiv) {
+            if (this.appData.documentation) {
+                docsDiv.innerHTML = this.renderMarkdown(this.appData.documentation);
+                // Add copy buttons to all code blocks
+                this.addCopyButtonsToCodeBlocks(docsDiv);
+            } else {
+                docsDiv.innerHTML = '<p>Documentation coming soon.</p>';
+            }
+        }
+    }
+
+    addCopyButtonsToCodeBlocks(container) {
+        // Find all code blocks and add copy buttons
+        const codeBlocks = container.querySelectorAll('pre code');
+        codeBlocks.forEach(codeBlock => {
+            const pre = codeBlock.parentElement;
+
+            // Skip if already has a copy button
+            if (pre.querySelector('.copy-btn')) return;
+
+            // Create copy button
+            const copyBtn = document.createElement('button');
+            copyBtn.className = 'copy-btn';
+            copyBtn.textContent = 'Copy';
+            copyBtn.onclick = () => {
+                navigator.clipboard.writeText(codeBlock.textContent).then(() => {
+                    copyBtn.textContent = '✓ Copied!';
+                    setTimeout(() => {
+                        copyBtn.textContent = 'Copy';
+                    }, 2000);
+                });
+            };
+
+            // Add button to pre element
+            pre.style.position = 'relative';
+            pre.insertBefore(copyBtn, codeBlock);
+        });
+    }
+
+    renderMarkdown(text) {
+        if (!text) return '';
+
+        // Store code blocks temporarily to protect them from processing
+        const codeBlocks = [];
+        let processed = text.replace(/```(\w+)?\n([\s\S]*?)```/g, (match, lang, code) => {
+            const placeholder = `___CODE_BLOCK_${codeBlocks.length}___`;
+            codeBlocks.push(`<pre><code class="language-${lang || ''}">${this.escapeHtml(code)}</code></pre>`);
+            return placeholder;
+        });
+
+        // Store inline code temporarily
+        const inlineCodes = [];
+        processed = processed.replace(/`([^`]+)`/g, (match, code) => {
+            const placeholder = `___INLINE_CODE_${inlineCodes.length}___`;
+            inlineCodes.push(`<code>${this.escapeHtml(code)}</code>`);
+            return placeholder;
+        });
+
+        // Now process the rest of the markdown
+        processed = processed
+            // Headers
+            .replace(/^### (.*$)/gim, '<h3>$1</h3>')
+            .replace(/^## (.*$)/gim, '<h2>$1</h2>')
+            .replace(/^# (.*$)/gim, '<h1>$1</h1>')
+            // Bold
+            .replace(/\*\*(.*?)\*\*/g, '<strong>$1</strong>')
+            // Italic
+            .replace(/\*(.*?)\*/g, '<em>$1</em>')
+            // Links
+            .replace(/\[([^\]]+)\]\(([^)]+)\)/g, '<a href="$2" target="_blank">$1</a>')
+            // Line breaks
+            .replace(/\n\n/g, '</p><p>')
+            .replace(/\n/g, '<br>')
+            // Lists
+            .replace(/^\* (.*)$/gim, '<li>$1</li>')
+            .replace(/^- (.*)$/gim, '<li>$1</li>')
+            // Wrap in paragraphs
+            .replace(/^(?!<[h|p|pre|ul|ol|li])/gim, '<p>')
+            .replace(/(?<![>])$/gim, '</p>');
+
+        // Restore inline code
+        inlineCodes.forEach((code, i) => {
+            processed = processed.replace(`___INLINE_CODE_${i}___`, code);
+        });
+
+        // Restore code blocks
+        codeBlocks.forEach((block, i) => {
+            processed = processed.replace(`___CODE_BLOCK_${i}___`, block);
+        });
+
+        return processed;
+    }
+
+    escapeHtml(text) {
+        const div = document.createElement('div');
+        div.textContent = text;
+        return div.innerHTML;
+    }
+
+    formatNumber(num) {
+        if (num >= 1000000) {
+            return (num / 1000000).toFixed(1) + 'M';
+        } else if (num >= 1000) {
+            return (num / 1000).toFixed(1) + 'K';
+        }
+        return num.toString();
+    }
+
+    setupEventListeners() {
+        // Tab switching
+        const tabs = document.querySelectorAll('.tab-btn');
+        tabs.forEach(tab => {
+            tab.addEventListener('click', () => {
+                // Update active tab
+                tabs.forEach(t => t.classList.remove('active'));
+                tab.classList.add('active');
+
+                // Show corresponding content
+                const tabName = tab.dataset.tab;
+                document.querySelectorAll('.tab-content').forEach(content => {
+                    content.classList.remove('active');
+                });
+                document.getElementById(`${tabName}-tab`).classList.add('active');
+            });
+        });
+    }
+
+    async loadRelatedApps() {
+        try {
+            const response = await fetch(`${API_BASE}/apps?category=${encodeURIComponent(this.appData.category)}&limit=4`);
+            const apps = await response.json();
+
+            const relatedApps = apps.filter(app => app.slug !== this.appSlug).slice(0, 3);
+            const grid = document.getElementById('related-apps-grid');
+
+            grid.innerHTML = relatedApps.map(app => `
+                <div class="related-app-card" onclick="window.location.href='app-detail.html?app=${app.slug || app.name.toLowerCase().replace(/\s+/g, '-')}'">
+                    <h4>${app.name}</h4>
+                    <p>${app.description.substring(0, 100)}...</p>
+                    <div style="display: flex; justify-content: space-between; margin-top: 0.5rem; font-size: 0.75rem;">
+                        <span style="color: var(--primary-cyan)">${app.type}</span>
+                        <span style="color: var(--warning)">★ ${app.rating}/5</span>
+                    </div>
+                </div>
+            `).join('');
+        } catch (error) {
+            console.error('Error loading related apps:', error);
+        }
+    }
+}
+
+// Initialize when DOM is loaded
+document.addEventListener('DOMContentLoaded', () => {
+    new AppDetailPage();
+});

docs/md_v2/marketplace/backend/.env.example

@@ -0,0 +1,14 @@
+# Marketplace Configuration
+# Copy this to .env and update with your values
+
+# Admin password (required)
+MARKETPLACE_ADMIN_PASSWORD=change_this_password
+
+# JWT secret key (required) - generate with: python3 -c "import secrets; print(secrets.token_urlsafe(32))"
+MARKETPLACE_JWT_SECRET=change_this_to_a_secure_random_key
+
+# Database path (optional, defaults to ./marketplace.db)
+MARKETPLACE_DB_PATH=./marketplace.db
+
+# Token expiry in hours (optional, defaults to 4)
+MARKETPLACE_TOKEN_EXPIRY=4

docs/md_v2/marketplace/backend/config.py

@@ -0,0 +1,59 @@
+"""
+Marketplace Configuration - Loads from .env file
+"""
+import os
+import sys
+import hashlib
+from pathlib import Path
+from dotenv import load_dotenv
+
+# Load .env file
+env_path = Path(__file__).parent / '.env'
+if not env_path.exists():
+    print("\n❌ ERROR: No .env file found!")
+    print("Please copy .env.example to .env and update with your values:")
+    print(f"  cp {Path(__file__).parent}/.env.example {Path(__file__).parent}/.env")
+    print("\nThen edit .env with your secure values.")
+    sys.exit(1)
+
+load_dotenv(env_path)
+
+# Required environment variables
+required_vars = ['MARKETPLACE_ADMIN_PASSWORD', 'MARKETPLACE_JWT_SECRET']
+missing_vars = [var for var in required_vars if not os.getenv(var)]
+
+if missing_vars:
+    print(f"\n❌ ERROR: Missing required environment variables: {', '.join(missing_vars)}")
+    print("Please check your .env file and ensure all required variables are set.")
+    sys.exit(1)
+
+class Config:
+    """Configuration loaded from environment variables"""
+
+    # Admin authentication - hashed from password in .env
+    ADMIN_PASSWORD_HASH = hashlib.sha256(
+        os.getenv('MARKETPLACE_ADMIN_PASSWORD').encode()
+    ).hexdigest()
+
+    # JWT secret for token generation
+    JWT_SECRET_KEY = os.getenv('MARKETPLACE_JWT_SECRET')
+
+    # Database path
+    DATABASE_PATH = os.getenv('MARKETPLACE_DB_PATH', './marketplace.db')
+
+    # Token expiry in hours
+    TOKEN_EXPIRY_HOURS = int(os.getenv('MARKETPLACE_TOKEN_EXPIRY', '4'))
+
+    # CORS origins - hardcoded as they don't contain secrets
+    ALLOWED_ORIGINS = [
+        "http://localhost:8000",
+        "http://localhost:8080",
+        "http://localhost:8100",
+        "http://127.0.0.1:8000",
+        "http://127.0.0.1:8080",
+        "http://127.0.0.1:8100",
+        "https://crawl4ai.com",
+        "https://www.crawl4ai.com",
+        "https://docs.crawl4ai.com",
+        "https://market.crawl4ai.com"
+    ]

docs/md_v2/marketplace/backend/database.py

@@ -0,0 +1,117 @@
+import sqlite3
+import yaml
+import json
+from pathlib import Path
+from typing import Dict, List, Any
+
+class DatabaseManager:
+    def __init__(self, db_path=None, schema_path='schema.yaml'):
+        self.schema = self._load_schema(schema_path)
+        # Use provided path or fallback to schema default
+        self.db_path = db_path or self.schema['database']['name']
+        self.conn = None
+        self._init_database()
+
+    def _load_schema(self, path: str) -> Dict:
+        with open(path, 'r') as f:
+            return yaml.safe_load(f)
+
+    def _init_database(self):
+        """Auto-create/migrate database from schema"""
+        self.conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        self.conn.row_factory = sqlite3.Row
+
+        for table_name, table_def in self.schema['tables'].items():
+            self._create_or_update_table(table_name, table_def['columns'])
+
+    def _create_or_update_table(self, table_name: str, columns: Dict):
+        cursor = self.conn.cursor()
+
+        # Check if table exists
+        cursor.execute(f"SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table_name,))
+        table_exists = cursor.fetchone() is not None
+
+        if not table_exists:
+            # Create table
+            col_defs = []
+            for col_name, col_spec in columns.items():
+                col_def = f"{col_name} {col_spec['type']}"
+                if col_spec.get('primary'):
+                    col_def += " PRIMARY KEY"
+                if col_spec.get('autoincrement'):
+                    col_def += " AUTOINCREMENT"
+                if col_spec.get('unique'):
+                    col_def += " UNIQUE"
+                if col_spec.get('required'):
+                    col_def += " NOT NULL"
+                if 'default' in col_spec:
+                    default = col_spec['default']
+                    if default == 'CURRENT_TIMESTAMP':
+                        col_def += f" DEFAULT {default}"
+                    elif isinstance(default, str):
+                        col_def += f" DEFAULT '{default}'"
+                    else:
+                        col_def += f" DEFAULT {default}"
+                col_defs.append(col_def)
+
+            create_sql = f"CREATE TABLE {table_name} ({', '.join(col_defs)})"
+            cursor.execute(create_sql)
+        else:
+            # Check for new columns and add them
+            cursor.execute(f"PRAGMA table_info({table_name})")
+            existing_columns = {row[1] for row in cursor.fetchall()}
+
+            for col_name, col_spec in columns.items():
+                if col_name not in existing_columns:
+                    col_def = f"{col_spec['type']}"
+                    if 'default' in col_spec:
+                        default = col_spec['default']
+                        if default == 'CURRENT_TIMESTAMP':
+                            col_def += f" DEFAULT {default}"
+                        elif isinstance(default, str):
+                            col_def += f" DEFAULT '{default}'"
+                        else:
+                            col_def += f" DEFAULT {default}"
+
+                    cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN {col_name} {col_def}")
+
+        self.conn.commit()
+
+    def get_all(self, table: str, limit: int = 100, offset: int = 0, where: str = None) -> List[Dict]:
+        cursor = self.conn.cursor()
+        query = f"SELECT * FROM {table}"
+        if where:
+            query += f" WHERE {where}"
+        query += f" LIMIT {limit} OFFSET {offset}"
+
+        cursor.execute(query)
+        rows = cursor.fetchall()
+        return [dict(row) for row in rows]
+
+    def search(self, query: str, tables: List[str] = None) -> Dict[str, List[Dict]]:
+        if not tables:
+            tables = list(self.schema['tables'].keys())
+
+        results = {}
+        cursor = self.conn.cursor()
+
+        for table in tables:
+            # Search in text columns
+            columns = self.schema['tables'][table]['columns']
+            text_cols = [col for col, spec in columns.items()
+                        if spec['type'] == 'TEXT' and col != 'id']
+
+            if text_cols:
+                where_clause = ' OR '.join([f"{col} LIKE ?" for col in text_cols])
+                params = [f'%{query}%'] * len(text_cols)
+
+                cursor.execute(f"SELECT * FROM {table} WHERE {where_clause} LIMIT 10", params)
+                rows = cursor.fetchall()
+                if rows:
+                    results[table] = [dict(row) for row in rows]
+
+        return results
+
+    def close(self):
+        if self.conn:
+            self.conn.close()

docs/md_v2/marketplace/backend/dummy_data.py

@@ -0,0 +1,267 @@
+import sqlite3
+import json
+import random
+from datetime import datetime, timedelta
+from database import DatabaseManager
+
+def generate_slug(text):
+    return text.lower().replace(' ', '-').replace('&', 'and')
+
+def generate_dummy_data():
+    db = DatabaseManager()
+    conn = db.conn
+    cursor = conn.cursor()
+
+    # Clear existing data
+    for table in ['apps', 'articles', 'categories', 'sponsors']:
+        cursor.execute(f"DELETE FROM {table}")
+
+    # Categories
+    categories = [
+        ("Browser Automation", "⚙", "Tools for browser automation and control"),
+        ("Proxy Services", "🔒", "Proxy providers and rotation services"),
+        ("LLM Integration", "🤖", "AI/LLM tools and integrations"),
+        ("Data Processing", "📊", "Data extraction and processing tools"),
+        ("Cloud Infrastructure", "☁", "Cloud browser and computing services"),
+        ("Developer Tools", "🛠", "Development and testing utilities")
+    ]
+
+    for i, (name, icon, desc) in enumerate(categories):
+        cursor.execute("""
+            INSERT INTO categories (name, slug, icon, description, order_index)
+            VALUES (?, ?, ?, ?, ?)
+        """, (name, generate_slug(name), icon, desc, i))
+
+    # Apps with real Unsplash images
+    apps_data = [
+        # Browser Automation
+        ("Playwright Cloud", "Browser Automation", "Paid", True, True,
+         "Scalable browser automation in the cloud with Playwright", "https://playwright.cloud",
+         None, "$99/month starter", 4.8, 12500,
+         "https://images.unsplash.com/photo-1633356122544-f134324a6cee?w=800&h=400&fit=crop"),
+
+        ("Selenium Grid Hub", "Browser Automation", "Freemium", False, False,
+         "Distributed Selenium grid for parallel testing", "https://seleniumhub.io",
+         "https://github.com/seleniumhub/grid", "Free - $299/month", 4.2, 8400,
+         "https://images.unsplash.com/photo-1555066931-4365d14bab8c?w=800&h=400&fit=crop"),
+
+        ("Puppeteer Extra", "Browser Automation", "Open Source", True, False,
+         "Enhanced Puppeteer with stealth plugins and more", "https://puppeteer-extra.dev",
+         "https://github.com/berstend/puppeteer-extra", "Free", 4.6, 15200,
+         "https://images.unsplash.com/photo-1461749280684-dccba630e2f6?w=800&h=400&fit=crop"),
+
+        # Proxy Services
+        ("BrightData", "Proxy Services", "Paid", True, True,
+         "Premium proxy network with 72M+ IPs worldwide", "https://brightdata.com",
+         None, "Starting $500/month", 4.7, 9800,
+         "https://images.unsplash.com/photo-1558494949-ef010cbdcc31?w=800&h=400&fit=crop"),
+
+        ("SmartProxy", "Proxy Services", "Paid", False, True,
+         "Residential and datacenter proxies with rotation", "https://smartproxy.com",
+         None, "Starting $75/month", 4.3, 7600,
+         "https://images.unsplash.com/photo-1544197150-b99a580bb7a8?w=800&h=400&fit=crop"),
+
+        ("ProxyMesh", "Proxy Services", "Freemium", False, False,
+         "Rotating proxy servers with sticky sessions", "https://proxymesh.com",
+         None, "$10-$50/month", 4.0, 4200,
+         "https://images.unsplash.com/photo-1451187580459-43490279c0fa?w=800&h=400&fit=crop"),
+
+        # LLM Integration
+        ("LangChain Crawl", "LLM Integration", "Open Source", True, False,
+         "LangChain integration for Crawl4AI workflows", "https://langchain-crawl.dev",
+         "https://github.com/langchain/crawl", "Free", 4.5, 18900,
+         "https://images.unsplash.com/photo-1677442136019-21780ecad995?w=800&h=400&fit=crop"),
+
+        ("GPT Scraper", "LLM Integration", "Freemium", False, False,
+         "Extract structured data using GPT models", "https://gptscraper.ai",
+         None, "Free - $99/month", 4.1, 5600,
+         "https://images.unsplash.com/photo-1655720828018-edd2daec9349?w=800&h=400&fit=crop"),
+
+        ("Claude Extract", "LLM Integration", "Paid", True, True,
+         "Professional extraction using Claude AI", "https://claude-extract.com",
+         None, "$199/month", 4.9, 3200,
+         "https://images.unsplash.com/photo-1686191128892-3b09ad503b4f?w=800&h=400&fit=crop"),
+
+        # Data Processing
+        ("DataMiner Pro", "Data Processing", "Paid", False, False,
+         "Advanced data extraction and transformation", "https://dataminer.pro",
+         None, "$149/month", 4.2, 6700,
+         "https://images.unsplash.com/photo-1551288049-bebda4e38f71?w=800&h=400&fit=crop"),
+
+        ("ScraperAPI", "Data Processing", "Freemium", True, True,
+         "Simple API for web scraping with proxy rotation", "https://scraperapi.com",
+         None, "Free - $299/month", 4.6, 22300,
+         "https://images.unsplash.com/photo-1460925895917-afdab827c52f?w=800&h=400&fit=crop"),
+
+        ("Apify", "Data Processing", "Freemium", False, False,
+         "Web scraping and automation platform", "https://apify.com",
+         None, "$49-$499/month", 4.4, 14500,
+         "https://images.unsplash.com/photo-1504639725590-34d0984388bd?w=800&h=400&fit=crop"),
+
+        # Cloud Infrastructure
+        ("BrowserCloud", "Cloud Infrastructure", "Paid", True, True,
+         "Managed headless browsers in the cloud", "https://browsercloud.io",
+         None, "$199/month", 4.5, 8900,
+         "https://images.unsplash.com/photo-1667372393119-3d4c48d07fc9?w=800&h=400&fit=crop"),
+
+        ("LambdaTest", "Cloud Infrastructure", "Freemium", False, False,
+         "Cross-browser testing on cloud", "https://lambdatest.com",
+         None, "Free - $99/month", 4.1, 11200,
+         "https://images.unsplash.com/photo-1451187580459-43490279c0fa?w=800&h=400&fit=crop"),
+
+        ("Browserless", "Cloud Infrastructure", "Freemium", True, False,
+         "Headless browser automation API", "https://browserless.io",
+         None, "$50-$500/month", 4.7, 19800,
+         "https://images.unsplash.com/photo-1639762681485-074b7f938ba0?w=800&h=400&fit=crop"),
+
+        # Developer Tools
+        ("Crawl4AI VSCode", "Developer Tools", "Open Source", True, False,
+         "VSCode extension for Crawl4AI development", "https://marketplace.visualstudio.com",
+         "https://github.com/crawl4ai/vscode", "Free", 4.8, 34500,
+         "https://images.unsplash.com/photo-1629654297299-c8506221ca97?w=800&h=400&fit=crop"),
+
+        ("Postman Collection", "Developer Tools", "Open Source", False, False,
+         "Postman collection for Crawl4AI API testing", "https://postman.com/crawl4ai",
+         "https://github.com/crawl4ai/postman", "Free", 4.3, 7800,
+         "https://images.unsplash.com/photo-1599507593499-a3f7d7d97667?w=800&h=400&fit=crop"),
+
+        ("Debug Toolkit", "Developer Tools", "Open Source", False, False,
+         "Debugging tools for crawler development", "https://debug.crawl4ai.com",
+         "https://github.com/crawl4ai/debug", "Free", 4.0, 4300,
+         "https://images.unsplash.com/photo-1515879218367-8466d910aaa4?w=800&h=400&fit=crop"),
+    ]
+
+    for name, category, type_, featured, sponsored, desc, url, github, pricing, rating, downloads, image in apps_data:
+        screenshots = json.dumps([
+            f"https://images.unsplash.com/photo-{random.randint(1500000000000, 1700000000000)}-{random.randint(1000000000000, 9999999999999)}?w=800&h=600&fit=crop",
+            f"https://images.unsplash.com/photo-{random.randint(1500000000000, 1700000000000)}-{random.randint(1000000000000, 9999999999999)}?w=800&h=600&fit=crop"
+        ])
+        cursor.execute("""
+            INSERT INTO apps (name, slug, description, category, type, featured, sponsored,
+                            website_url, github_url, pricing, rating, downloads, image, screenshots, logo_url,
+                            integration_guide, contact_email, views)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        """, (name, generate_slug(name), desc, category, type_, featured, sponsored,
+             url, github, pricing, rating, downloads, image, screenshots,
+             f"https://ui-avatars.com/api/?name={name}&background=50ffff&color=070708&size=128",
+             f"# {name} Integration\n\n```python\nfrom crawl4ai import AsyncWebCrawler\n# Integration code coming soon...\n```",
+             f"contact@{generate_slug(name)}.com",
+             random.randint(100, 5000)))
+
+    # Articles with real images
+    articles_data = [
+        ("Browser Automation Showdown: Playwright vs Puppeteer vs Selenium",
+         "Review", "John Doe", ["Playwright Cloud", "Puppeteer Extra"],
+         ["browser-automation", "comparison", "2024"],
+         "https://images.unsplash.com/photo-1587620962725-abab7fe55159?w=1200&h=630&fit=crop"),
+
+        ("Top 5 Proxy Services for Web Scraping in 2024",
+         "Comparison", "Jane Smith", ["BrightData", "SmartProxy", "ProxyMesh"],
+         ["proxy", "web-scraping", "guide"],
+         "https://images.unsplash.com/photo-1558494949-ef010cbdcc31?w=1200&h=630&fit=crop"),
+
+        ("Integrating LLMs with Crawl4AI: A Complete Guide",
+         "Tutorial", "Crawl4AI Team", ["LangChain Crawl", "GPT Scraper", "Claude Extract"],
+         ["llm", "integration", "tutorial"],
+         "https://images.unsplash.com/photo-1677442136019-21780ecad995?w=1200&h=630&fit=crop"),
+
+        ("Building Scalable Crawlers with Cloud Infrastructure",
+         "Tutorial", "Mike Johnson", ["BrowserCloud", "Browserless"],
+         ["cloud", "scalability", "architecture"],
+         "https://images.unsplash.com/photo-1667372393119-3d4c48d07fc9?w=1200&h=630&fit=crop"),
+
+        ("What's New in Crawl4AI Marketplace",
+         "News", "Crawl4AI Team", [],
+         ["marketplace", "announcement", "news"],
+         "https://images.unsplash.com/photo-1556075798-4825dfaaf498?w=1200&h=630&fit=crop"),
+
+        ("Cost Analysis: Self-Hosted vs Cloud Browser Solutions",
+         "Comparison", "Sarah Chen", ["BrowserCloud", "LambdaTest", "Browserless"],
+         ["cost", "cloud", "comparison"],
+         "https://images.unsplash.com/photo-1554224155-8d04cb21cd6c?w=1200&h=630&fit=crop"),
+
+        ("Getting Started with Browser Automation",
+         "Tutorial", "Crawl4AI Team", ["Playwright Cloud", "Selenium Grid Hub"],
+         ["beginner", "tutorial", "automation"],
+         "https://images.unsplash.com/photo-1498050108023-c5249f4df085?w=1200&h=630&fit=crop"),
+
+        ("The Future of Web Scraping: AI-Powered Extraction",
+         "News", "Dr. Alan Turing", ["Claude Extract", "GPT Scraper"],
+         ["ai", "future", "trends"],
+         "https://images.unsplash.com/photo-1593720213428-28a5b9e94613?w=1200&h=630&fit=crop")
+    ]
+
+    for title, category, author, related_apps, tags, image in articles_data:
+        # Get app IDs for related apps
+        related_ids = []
+        for app_name in related_apps:
+            cursor.execute("SELECT id FROM apps WHERE name = ?", (app_name,))
+            result = cursor.fetchone()
+            if result:
+                related_ids.append(result[0])
+
+        content = f"""# {title}
+
+By {author} | {datetime.now().strftime('%B %d, %Y')}
+
+## Introduction
+
+This is a comprehensive article about {title.lower()}. Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+
+## Key Points
+
+- Important point about the topic
+- Another crucial insight
+- Technical details and specifications
+- Performance comparisons
+
+## Conclusion
+
+In summary, this article explored various aspects of the topic. Stay tuned for more updates!
+"""
+
+        cursor.execute("""
+            INSERT INTO articles (title, slug, content, author, category, related_apps,
+                                featured_image, tags, views)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+        """, (title, generate_slug(title), content, author, category,
+             json.dumps(related_ids), image, json.dumps(tags),
+             random.randint(200, 10000)))
+
+    # Sponsors
+    sponsors_data = [
+        ("BrightData", "Gold", "https://brightdata.com",
+         "https://images.unsplash.com/photo-1558494949-ef010cbdcc31?w=728&h=90&fit=crop"),
+        ("ScraperAPI", "Gold", "https://scraperapi.com",
+         "https://images.unsplash.com/photo-1460925895917-afdab827c52f?w=728&h=90&fit=crop"),
+        ("BrowserCloud", "Silver", "https://browsercloud.io",
+         "https://images.unsplash.com/photo-1667372393119-3d4c48d07fc9?w=728&h=90&fit=crop"),
+        ("Claude Extract", "Silver", "https://claude-extract.com",
+         "https://images.unsplash.com/photo-1686191128892-3b09ad503b4f?w=728&h=90&fit=crop"),
+        ("SmartProxy", "Bronze", "https://smartproxy.com",
+         "https://images.unsplash.com/photo-1544197150-b99a580bb7a8?w=728&h=90&fit=crop")
+    ]
+
+    for company, tier, landing_url, banner in sponsors_data:
+        start_date = datetime.now() - timedelta(days=random.randint(1, 30))
+        end_date = datetime.now() + timedelta(days=random.randint(30, 180))
+
+        cursor.execute("""
+            INSERT INTO sponsors (company_name, logo_url, tier, banner_url,
+                                landing_url, active, start_date, end_date)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+        """, (company,
+             f"https://ui-avatars.com/api/?name={company}&background=09b5a5&color=fff&size=200",
+             tier, banner, landing_url, 1,
+             start_date.isoformat(), end_date.isoformat()))
+
+    conn.commit()
+    print("✓ Dummy data generated successfully!")
+    print(f"  - {len(categories)} categories")
+    print(f"  - {len(apps_data)} apps")
+    print(f"  - {len(articles_data)} articles")
+    print(f"  - {len(sponsors_data)} sponsors")
+
+if __name__ == "__main__":
+    generate_dummy_data()

docs/md_v2/marketplace/backend/requirements.txt

@@ -0,0 +1,5 @@
+fastapi
+uvicorn
+pyyaml
+python-multipart
+python-dotenv

docs/md_v2/marketplace/backend/schema.yaml

@@ -0,0 +1,75 @@
+database:
+  name: marketplace.db
+
+tables:
+  apps:
+    columns:
+      id: {type: INTEGER, primary: true, autoincrement: true}
+      name: {type: TEXT, required: true}
+      slug: {type: TEXT, unique: true}
+      description: {type: TEXT}
+      long_description: {type: TEXT}
+      logo_url: {type: TEXT}
+      image: {type: TEXT}
+      screenshots: {type: JSON, default: '[]'}
+      category: {type: TEXT}
+      type: {type: TEXT, default: 'Open Source'}
+      status: {type: TEXT, default: 'Active'}
+      website_url: {type: TEXT}
+      github_url: {type: TEXT}
+      demo_url: {type: TEXT}
+      video_url: {type: TEXT}
+      documentation_url: {type: TEXT}
+      support_url: {type: TEXT}
+      discord_url: {type: TEXT}
+      pricing: {type: TEXT}
+      rating: {type: REAL, default: 0.0}
+      downloads: {type: INTEGER, default: 0}
+      featured: {type: BOOLEAN, default: 0}
+      sponsored: {type: BOOLEAN, default: 0}
+      integration_guide: {type: TEXT}
+      documentation: {type: TEXT}
+      examples: {type: TEXT}
+      installation_command: {type: TEXT}
+      requirements: {type: TEXT}
+      changelog: {type: TEXT}
+      tags: {type: JSON, default: '[]'}
+      added_date: {type: DATETIME, default: CURRENT_TIMESTAMP}
+      updated_date: {type: DATETIME, default: CURRENT_TIMESTAMP}
+      contact_email: {type: TEXT}
+      views: {type: INTEGER, default: 0}
+
+  articles:
+    columns:
+      id: {type: INTEGER, primary: true, autoincrement: true}
+      title: {type: TEXT, required: true}
+      slug: {type: TEXT, unique: true}
+      content: {type: TEXT}
+      author: {type: TEXT, default: 'Crawl4AI Team'}
+      category: {type: TEXT}
+      related_apps: {type: JSON, default: '[]'}
+      featured_image: {type: TEXT}
+      published_date: {type: DATETIME, default: CURRENT_TIMESTAMP}
+      tags: {type: JSON, default: '[]'}
+      views: {type: INTEGER, default: 0}
+
+  categories:
+    columns:
+      id: {type: INTEGER, primary: true, autoincrement: true}
+      name: {type: TEXT, unique: true}
+      slug: {type: TEXT, unique: true}
+      icon: {type: TEXT}
+      description: {type: TEXT}
+      order_index: {type: INTEGER, default: 0}
+
+  sponsors:
+    columns:
+      id: {type: INTEGER, primary: true, autoincrement: true}
+      company_name: {type: TEXT, required: true}
+      logo_url: {type: TEXT}
+      tier: {type: TEXT, default: 'Bronze'}
+      banner_url: {type: TEXT}
+      landing_url: {type: TEXT}
+      active: {type: BOOLEAN, default: 1}
+      start_date: {type: DATETIME}
+      end_date: {type: DATETIME}

docs/md_v2/marketplace/backend/server.py

@@ -0,0 +1,493 @@
+from fastapi import FastAPI, HTTPException, Query, Depends, Body, UploadFile, File, Form, APIRouter
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from typing import Optional, Dict, Any
+import json
+import hashlib
+import secrets
+import re
+from pathlib import Path
+from database import DatabaseManager
+from datetime import datetime, timedelta
+
+# Import configuration (will exit if .env not found or invalid)
+from config import Config
+
+app = FastAPI(title="Crawl4AI Marketplace API")
+router = APIRouter(prefix="/marketplace/api")
+
+# Security setup
+security = HTTPBearer()
+tokens = {}  # In production, use Redis or database for token storage
+
+# CORS configuration
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=Config.ALLOWED_ORIGINS,
+    allow_credentials=True,
+    allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
+    allow_headers=["*"],
+    max_age=3600
+)
+
+# Initialize database with configurable path
+db = DatabaseManager(Config.DATABASE_PATH)
+
+BASE_DIR = Path(__file__).parent
+UPLOAD_ROOT = BASE_DIR / "uploads"
+UPLOAD_ROOT.mkdir(parents=True, exist_ok=True)
+
+app.mount("/uploads", StaticFiles(directory=UPLOAD_ROOT), name="uploads")
+
+ALLOWED_IMAGE_TYPES = {
+    "image/png": ".png",
+    "image/jpeg": ".jpg",
+    "image/webp": ".webp",
+    "image/svg+xml": ".svg"
+}
+ALLOWED_UPLOAD_FOLDERS = {"sponsors"}
+MAX_UPLOAD_SIZE = 2 * 1024 * 1024  # 2 MB
+
+def json_response(data, cache_time=3600):
+    """Helper to return JSON with cache headers"""
+    return JSONResponse(
+        content=data,
+        headers={
+            "Cache-Control": f"public, max-age={cache_time}",
+            "X-Content-Type-Options": "nosniff"
+        }
+    )
+
+
+def to_int(value, default=0):
+    """Coerce incoming values to integers, falling back to default."""
+    if value is None:
+        return default
+    if isinstance(value, bool):
+        return int(value)
+    if isinstance(value, (int, float)):
+        return int(value)
+
+    if isinstance(value, str):
+        stripped = value.strip()
+        if not stripped:
+            return default
+
+        match = re.match(r"^-?\d+", stripped)
+        if match:
+            try:
+                return int(match.group())
+            except ValueError:
+                return default
+    return default
+
+# ============= PUBLIC ENDPOINTS =============
+
+@router.get("/apps")
+async def get_apps(
+    category: Optional[str] = None,
+    type: Optional[str] = None,
+    featured: Optional[bool] = None,
+    sponsored: Optional[bool] = None,
+    limit: int = Query(default=20, le=10000),
+    offset: int = Query(default=0)
+):
+    """Get apps with optional filters"""
+    where_clauses = []
+    if category:
+        where_clauses.append(f"category = '{category}'")
+    if type:
+        where_clauses.append(f"type = '{type}'")
+    if featured is not None:
+        where_clauses.append(f"featured = {1 if featured else 0}")
+    if sponsored is not None:
+        where_clauses.append(f"sponsored = {1 if sponsored else 0}")
+
+    where = " AND ".join(where_clauses) if where_clauses else None
+    apps = db.get_all('apps', limit=limit, offset=offset, where=where)
+
+    # Parse JSON fields
+    for app in apps:
+        if app.get('screenshots'):
+            app['screenshots'] = json.loads(app['screenshots'])
+
+    return json_response(apps)
+
+@router.get("/apps/{slug}")
+async def get_app(slug: str):
+    """Get single app by slug"""
+    apps = db.get_all('apps', where=f"slug = '{slug}'", limit=1)
+    if not apps:
+        raise HTTPException(status_code=404, detail="App not found")
+
+    app = apps[0]
+    if app.get('screenshots'):
+        app['screenshots'] = json.loads(app['screenshots'])
+
+    return json_response(app)
+
+@router.get("/articles")
+async def get_articles(
+    category: Optional[str] = None,
+    limit: int = Query(default=20, le=10000),
+    offset: int = Query(default=0)
+):
+    """Get articles with optional category filter"""
+    where = f"category = '{category}'" if category else None
+    articles = db.get_all('articles', limit=limit, offset=offset, where=where)
+
+    # Parse JSON fields
+    for article in articles:
+        if article.get('related_apps'):
+            article['related_apps'] = json.loads(article['related_apps'])
+        if article.get('tags'):
+            article['tags'] = json.loads(article['tags'])
+
+    return json_response(articles)
+
+@router.get("/articles/{slug}")
+async def get_article(slug: str):
+    """Get single article by slug"""
+    articles = db.get_all('articles', where=f"slug = '{slug}'", limit=1)
+    if not articles:
+        raise HTTPException(status_code=404, detail="Article not found")
+
+    article = articles[0]
+    if article.get('related_apps'):
+        article['related_apps'] = json.loads(article['related_apps'])
+    if article.get('tags'):
+        article['tags'] = json.loads(article['tags'])
+
+    return json_response(article)
+
+@router.get("/categories")
+async def get_categories():
+    """Get all categories ordered by index"""
+    categories = db.get_all('categories', limit=50)
+    for category in categories:
+        category['order_index'] = to_int(category.get('order_index'), 0)
+    categories.sort(key=lambda x: x.get('order_index', 0))
+    return json_response(categories, cache_time=7200)
+
+@router.get("/sponsors")
+async def get_sponsors(active: Optional[bool] = True):
+    """Get sponsors, default active only"""
+    where = f"active = {1 if active else 0}" if active is not None else None
+    sponsors = db.get_all('sponsors', where=where, limit=20)
+
+    # Filter by date if active
+    if active:
+        now = datetime.now().isoformat()
+        sponsors = [s for s in sponsors
+                   if (not s.get('start_date') or s['start_date'] <= now) and
+                      (not s.get('end_date') or s['end_date'] >= now)]
+
+    return json_response(sponsors)
+
+@router.get("/search")
+async def search(q: str = Query(min_length=2)):
+    """Search across apps and articles"""
+    if len(q) < 2:
+        return json_response({})
+
+    results = db.search(q, tables=['apps', 'articles'])
+
+    # Parse JSON fields in results
+    for table, items in results.items():
+        for item in items:
+            if table == 'apps' and item.get('screenshots'):
+                item['screenshots'] = json.loads(item['screenshots'])
+            elif table == 'articles':
+                if item.get('related_apps'):
+                    item['related_apps'] = json.loads(item['related_apps'])
+                if item.get('tags'):
+                    item['tags'] = json.loads(item['tags'])
+
+    return json_response(results, cache_time=1800)
+
+@router.get("/stats")
+async def get_stats():
+    """Get marketplace statistics"""
+    stats = {
+        "total_apps": len(db.get_all('apps', limit=10000)),
+        "total_articles": len(db.get_all('articles', limit=10000)),
+        "total_categories": len(db.get_all('categories', limit=1000)),
+        "active_sponsors": len(db.get_all('sponsors', where="active = 1", limit=1000))
+    }
+    return json_response(stats, cache_time=1800)
+
+# ============= ADMIN AUTHENTICATION =============
+
+def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)):
+    """Verify admin authentication token"""
+    token = credentials.credentials
+    if token not in tokens or tokens[token] < datetime.now():
+        raise HTTPException(status_code=401, detail="Invalid or expired token")
+    return token
+
+
+@router.post("/admin/upload-image", dependencies=[Depends(verify_token)])
+async def upload_image(file: UploadFile = File(...), folder: str = Form("sponsors")):
+    """Upload image files for admin assets"""
+    folder = (folder or "").strip().lower()
+    if folder not in ALLOWED_UPLOAD_FOLDERS:
+        raise HTTPException(status_code=400, detail="Invalid upload folder")
+
+    if file.content_type not in ALLOWED_IMAGE_TYPES:
+        raise HTTPException(status_code=400, detail="Unsupported file type")
+
+    contents = await file.read()
+    if len(contents) > MAX_UPLOAD_SIZE:
+        raise HTTPException(status_code=400, detail="File too large (max 2MB)")
+
+    extension = ALLOWED_IMAGE_TYPES[file.content_type]
+    filename = f"{datetime.now().strftime('%Y%m%d%H%M%S')}_{secrets.token_hex(8)}{extension}"
+
+    target_dir = UPLOAD_ROOT / folder
+    target_dir.mkdir(parents=True, exist_ok=True)
+    target_path = target_dir / filename
+    target_path.write_bytes(contents)
+
+    return {"url": f"/uploads/{folder}/{filename}"}
+
+@router.post("/admin/login")
+async def admin_login(password: str = Body(..., embed=True)):
+    """Admin login with password"""
+    provided_hash = hashlib.sha256(password.encode()).hexdigest()
+
+    if provided_hash != Config.ADMIN_PASSWORD_HASH:
+        # Log failed attempt in production
+        print(f"Failed login attempt at {datetime.now()}")
+        raise HTTPException(status_code=401, detail="Invalid password")
+
+    # Generate secure token
+    token = secrets.token_urlsafe(32)
+    tokens[token] = datetime.now() + timedelta(hours=Config.TOKEN_EXPIRY_HOURS)
+
+    return {
+        "token": token,
+        "expires_in": Config.TOKEN_EXPIRY_HOURS * 3600
+    }
+
+# ============= ADMIN ENDPOINTS =============
+
+@router.get("/admin/stats", dependencies=[Depends(verify_token)])
+async def get_admin_stats():
+    """Get detailed admin statistics"""
+    stats = {
+        "apps": {
+            "total": len(db.get_all('apps', limit=10000)),
+            "featured": len(db.get_all('apps', where="featured = 1", limit=10000)),
+            "sponsored": len(db.get_all('apps', where="sponsored = 1", limit=10000))
+        },
+        "articles": len(db.get_all('articles', limit=10000)),
+        "categories": len(db.get_all('categories', limit=1000)),
+        "sponsors": {
+            "active": len(db.get_all('sponsors', where="active = 1", limit=1000)),
+            "total": len(db.get_all('sponsors', limit=10000))
+        },
+        "total_views": sum(app.get('views', 0) for app in db.get_all('apps', limit=10000))
+    }
+    return stats
+
+# Apps CRUD
+@router.post("/admin/apps", dependencies=[Depends(verify_token)])
+async def create_app(app_data: Dict[str, Any]):
+    """Create new app"""
+    try:
+        # Handle JSON fields
+        for field in ['screenshots', 'tags']:
+            if field in app_data and isinstance(app_data[field], list):
+                app_data[field] = json.dumps(app_data[field])
+
+        cursor = db.conn.cursor()
+        columns = ', '.join(app_data.keys())
+        placeholders = ', '.join(['?' for _ in app_data])
+        cursor.execute(f"INSERT INTO apps ({columns}) VALUES ({placeholders})",
+                      list(app_data.values()))
+        db.conn.commit()
+        return {"id": cursor.lastrowid, "message": "App created"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@router.put("/admin/apps/{app_id}", dependencies=[Depends(verify_token)])
+async def update_app(app_id: int, app_data: Dict[str, Any]):
+    """Update app"""
+    try:
+        # Handle JSON fields
+        for field in ['screenshots', 'tags']:
+            if field in app_data and isinstance(app_data[field], list):
+                app_data[field] = json.dumps(app_data[field])
+
+        set_clause = ', '.join([f"{k} = ?" for k in app_data.keys()])
+        cursor = db.conn.cursor()
+        cursor.execute(f"UPDATE apps SET {set_clause} WHERE id = ?",
+                      list(app_data.values()) + [app_id])
+        db.conn.commit()
+        return {"message": "App updated"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@router.delete("/admin/apps/{app_id}", dependencies=[Depends(verify_token)])
+async def delete_app(app_id: int):
+    """Delete app"""
+    cursor = db.conn.cursor()
+    cursor.execute("DELETE FROM apps WHERE id = ?", (app_id,))
+    db.conn.commit()
+    return {"message": "App deleted"}
+
+# Articles CRUD
+@router.post("/admin/articles", dependencies=[Depends(verify_token)])
+async def create_article(article_data: Dict[str, Any]):
+    """Create new article"""
+    try:
+        for field in ['related_apps', 'tags']:
+            if field in article_data and isinstance(article_data[field], list):
+                article_data[field] = json.dumps(article_data[field])
+
+        cursor = db.conn.cursor()
+        columns = ', '.join(article_data.keys())
+        placeholders = ', '.join(['?' for _ in article_data])
+        cursor.execute(f"INSERT INTO articles ({columns}) VALUES ({placeholders})",
+                      list(article_data.values()))
+        db.conn.commit()
+        return {"id": cursor.lastrowid, "message": "Article created"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@router.put("/admin/articles/{article_id}", dependencies=[Depends(verify_token)])
+async def update_article(article_id: int, article_data: Dict[str, Any]):
+    """Update article"""
+    try:
+        for field in ['related_apps', 'tags']:
+            if field in article_data and isinstance(article_data[field], list):
+                article_data[field] = json.dumps(article_data[field])
+
+        set_clause = ', '.join([f"{k} = ?" for k in article_data.keys()])
+        cursor = db.conn.cursor()
+        cursor.execute(f"UPDATE articles SET {set_clause} WHERE id = ?",
+                      list(article_data.values()) + [article_id])
+        db.conn.commit()
+        return {"message": "Article updated"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@router.delete("/admin/articles/{article_id}", dependencies=[Depends(verify_token)])
+async def delete_article(article_id: int):
+    """Delete article"""
+    cursor = db.conn.cursor()
+    cursor.execute("DELETE FROM articles WHERE id = ?", (article_id,))
+    db.conn.commit()
+    return {"message": "Article deleted"}
+
+# Categories CRUD
+@router.post("/admin/categories", dependencies=[Depends(verify_token)])
+async def create_category(category_data: Dict[str, Any]):
+    """Create new category"""
+    try:
+        category_data = dict(category_data)
+        category_data['order_index'] = to_int(category_data.get('order_index'), 0)
+
+        cursor = db.conn.cursor()
+        columns = ', '.join(category_data.keys())
+        placeholders = ', '.join(['?' for _ in category_data])
+        cursor.execute(f"INSERT INTO categories ({columns}) VALUES ({placeholders})",
+                      list(category_data.values()))
+        db.conn.commit()
+        return {"id": cursor.lastrowid, "message": "Category created"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@router.put("/admin/categories/{cat_id}", dependencies=[Depends(verify_token)])
+async def update_category(cat_id: int, category_data: Dict[str, Any]):
+    """Update category"""
+    try:
+        category_data = dict(category_data)
+        if 'order_index' in category_data:
+            category_data['order_index'] = to_int(category_data.get('order_index'), 0)
+
+        set_clause = ', '.join([f"{k} = ?" for k in category_data.keys()])
+        cursor = db.conn.cursor()
+        cursor.execute(f"UPDATE categories SET {set_clause} WHERE id = ?",
+                      list(category_data.values()) + [cat_id])
+        db.conn.commit()
+        return {"message": "Category updated"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@router.delete("/admin/categories/{cat_id}", dependencies=[Depends(verify_token)])
+async def delete_category(cat_id: int):
+    """Delete category"""
+    try:
+        cursor = db.conn.cursor()
+        cursor.execute("DELETE FROM categories WHERE id = ?", (cat_id,))
+        db.conn.commit()
+        return {"message": "Category deleted"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+# Sponsors CRUD
+@router.post("/admin/sponsors", dependencies=[Depends(verify_token)])
+async def create_sponsor(sponsor_data: Dict[str, Any]):
+    """Create new sponsor"""
+    try:
+        cursor = db.conn.cursor()
+        columns = ', '.join(sponsor_data.keys())
+        placeholders = ', '.join(['?' for _ in sponsor_data])
+        cursor.execute(f"INSERT INTO sponsors ({columns}) VALUES ({placeholders})",
+                      list(sponsor_data.values()))
+        db.conn.commit()
+        return {"id": cursor.lastrowid, "message": "Sponsor created"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@router.put("/admin/sponsors/{sponsor_id}", dependencies=[Depends(verify_token)])
+async def update_sponsor(sponsor_id: int, sponsor_data: Dict[str, Any]):
+    """Update sponsor"""
+    try:
+        set_clause = ', '.join([f"{k} = ?" for k in sponsor_data.keys()])
+        cursor = db.conn.cursor()
+        cursor.execute(f"UPDATE sponsors SET {set_clause} WHERE id = ?",
+                      list(sponsor_data.values()) + [sponsor_id])
+        db.conn.commit()
+        return {"message": "Sponsor updated"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@router.delete("/admin/sponsors/{sponsor_id}", dependencies=[Depends(verify_token)])
+async def delete_sponsor(sponsor_id: int):
+    """Delete sponsor"""
+    try:
+        cursor = db.conn.cursor()
+        cursor.execute("DELETE FROM sponsors WHERE id = ?", (sponsor_id,))
+        db.conn.commit()
+        return {"message": "Sponsor deleted"}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+app.include_router(router)
+
+
+@app.get("/")
+async def root():
+    """API info"""
+    return {
+        "name": "Crawl4AI Marketplace API",
+        "version": "1.0.0",
+        "endpoints": [
+            "/marketplace/api/apps",
+            "/marketplace/api/articles",
+            "/marketplace/api/categories",
+            "/marketplace/api/sponsors",
+            "/marketplace/api/search?q=query",
+            "/marketplace/api/stats"
+        ]
+    }
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="127.0.0.1", port=8100)

docs/md_v2/marketplace/backend/uploads/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

docs/md_v2/marketplace/frontend/app-detail.css

@@ -0,0 +1,462 @@
+/* App Detail Page Styles */
+
+.app-detail-container {
+    min-height: 100vh;
+    background: var(--bg-dark);
+}
+
+/* Back Button */
+.header-nav {
+    display: flex;
+    align-items: center;
+}
+
+.back-btn {
+    padding: 0.5rem 1rem;
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--primary-cyan);
+    text-decoration: none;
+    transition: all 0.2s;
+    font-size: 0.875rem;
+}
+
+.back-btn:hover {
+    border-color: var(--primary-cyan);
+    background: rgba(80, 255, 255, 0.1);
+}
+
+/* App Hero Section */
+.app-hero {
+    max-width: 1800px;
+    margin: 2rem auto;
+    padding: 0 2rem;
+}
+
+.app-hero-content {
+    display: grid;
+    grid-template-columns: 1fr 2fr;
+    gap: 3rem;
+    background: linear-gradient(135deg, #1a1a2e, #0f0f1e);
+    border: 2px solid var(--primary-cyan);
+    padding: 2rem;
+    box-shadow: 0 0 30px rgba(80, 255, 255, 0.15),
+                inset 0 0 20px rgba(80, 255, 255, 0.05);
+}
+
+.app-hero-image {
+    width: 100%;
+    height: 300px;
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.1), rgba(243, 128, 245, 0.05));
+    background-size: cover;
+    background-position: center;
+    border: 1px solid var(--border-color);
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-size: 4rem;
+    color: var(--primary-cyan);
+}
+
+.app-badges {
+    display: flex;
+    gap: 0.5rem;
+    margin-bottom: 1rem;
+}
+
+.app-badge {
+    padding: 0.3rem 0.6rem;
+    background: var(--bg-tertiary);
+    color: var(--text-secondary);
+    font-size: 0.75rem;
+    text-transform: uppercase;
+    font-weight: 600;
+}
+
+.app-badge.featured {
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    color: var(--bg-dark);
+    box-shadow: 0 2px 10px rgba(80, 255, 255, 0.3);
+}
+
+.app-badge.sponsored {
+    background: linear-gradient(135deg, var(--warning), #ff8c00);
+    color: var(--bg-dark);
+    box-shadow: 0 2px 10px rgba(245, 158, 11, 0.3);
+}
+
+.app-hero-info h1 {
+    font-size: 2.5rem;
+    color: var(--primary-cyan);
+    margin: 0.5rem 0;
+    text-shadow: 0 0 20px rgba(80, 255, 255, 0.5);
+}
+
+.app-tagline {
+    font-size: 1.1rem;
+    color: var(--text-secondary);
+    margin-bottom: 2rem;
+}
+
+/* Stats */
+.app-stats {
+    display: flex;
+    gap: 2rem;
+    margin: 2rem 0;
+    padding: 1rem 0;
+    border-top: 1px solid var(--border-color);
+    border-bottom: 1px solid var(--border-color);
+}
+
+.stat {
+    display: flex;
+    flex-direction: column;
+    gap: 0.25rem;
+}
+
+.stat-value {
+    font-size: 1.5rem;
+    color: var(--primary-cyan);
+    font-weight: 600;
+}
+
+.stat-label {
+    font-size: 0.875rem;
+    color: var(--text-tertiary);
+}
+
+/* Action Buttons */
+.app-actions {
+    display: flex;
+    gap: 1rem;
+    margin: 2rem 0;
+}
+
+.action-btn {
+    padding: 0.75rem 1.5rem;
+    border: 1px solid var(--border-color);
+    background: transparent;
+    color: var(--text-primary);
+    text-decoration: none;
+    display: inline-flex;
+    align-items: center;
+    gap: 0.5rem;
+    transition: all 0.2s;
+    cursor: pointer;
+    font-family: inherit;
+    font-size: 0.9rem;
+}
+
+.action-btn.primary {
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    color: var(--bg-dark);
+    border-color: var(--primary-cyan);
+    font-weight: 600;
+}
+
+.action-btn.primary:hover {
+    box-shadow: 0 4px 15px rgba(80, 255, 255, 0.3);
+    transform: translateY(-2px);
+}
+
+.action-btn.secondary {
+    border-color: var(--accent-pink);
+    color: var(--accent-pink);
+}
+
+.action-btn.secondary:hover {
+    background: rgba(243, 128, 245, 0.1);
+    box-shadow: 0 4px 15px rgba(243, 128, 245, 0.2);
+}
+
+.action-btn.ghost {
+    border-color: var(--border-color);
+    color: var(--text-secondary);
+}
+
+.action-btn.ghost:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+/* Pricing */
+.pricing-info {
+    display: flex;
+    align-items: center;
+    gap: 1rem;
+    font-size: 1.1rem;
+}
+
+.pricing-label {
+    color: var(--text-tertiary);
+}
+
+.pricing-value {
+    color: var(--warning);
+    font-weight: 600;
+}
+
+/* Navigation Tabs */
+.app-nav {
+    max-width: 1800px;
+    margin: 2rem auto 0;
+    padding: 0 2rem;
+    display: flex;
+    gap: 1rem;
+    border-bottom: 2px solid var(--border-color);
+}
+
+.nav-tab {
+    padding: 1rem 1.5rem;
+    background: transparent;
+    border: none;
+    border-bottom: 2px solid transparent;
+    color: var(--text-secondary);
+    cursor: pointer;
+    transition: all 0.2s;
+    font-family: inherit;
+    font-size: 0.9rem;
+    margin-bottom: -2px;
+}
+
+.nav-tab:hover {
+    color: var(--primary-cyan);
+}
+
+.nav-tab.active {
+    color: var(--primary-cyan);
+    border-bottom-color: var(--primary-cyan);
+}
+
+/* Content Sections */
+.app-content {
+    max-width: 1800px;
+    margin: 2rem auto;
+    padding: 0 2rem;
+}
+
+.tab-content {
+    display: none;
+}
+
+.tab-content.active {
+    display: block;
+}
+
+.docs-content {
+    max-width: 1200px;
+    padding: 2rem;
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+}
+
+.docs-content h2 {
+    font-size: 1.8rem;
+    color: var(--primary-cyan);
+    margin-bottom: 1rem;
+    padding-bottom: 0.5rem;
+    border-bottom: 1px solid var(--border-color);
+}
+
+.docs-content h3 {
+    font-size: 1.3rem;
+    color: var(--text-primary);
+    margin: 2rem 0 1rem;
+}
+
+.docs-content h4 {
+    font-size: 1.1rem;
+    color: var(--accent-pink);
+    margin: 1.5rem 0 0.5rem;
+}
+
+.docs-content p {
+    color: var(--text-secondary);
+    line-height: 1.6;
+    margin-bottom: 1rem;
+}
+
+.docs-content code {
+    background: var(--bg-tertiary);
+    padding: 0.2rem 0.4rem;
+    color: var(--primary-cyan);
+    font-family: 'Dank Mono', Monaco, monospace;
+    font-size: 0.9em;
+}
+
+/* Code Blocks */
+.code-block {
+    background: var(--bg-dark);
+    border: 1px solid var(--border-color);
+    margin: 1rem 0;
+    overflow: hidden;
+}
+
+.code-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    padding: 0.5rem 1rem;
+    background: var(--bg-tertiary);
+    border-bottom: 1px solid var(--border-color);
+}
+
+.code-lang {
+    color: var(--primary-cyan);
+    font-size: 0.875rem;
+    text-transform: uppercase;
+}
+
+.copy-btn {
+    padding: 0.25rem 0.5rem;
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    cursor: pointer;
+    font-size: 0.75rem;
+    transition: all 0.2s;
+}
+
+.copy-btn:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+.code-block pre {
+    margin: 0;
+    padding: 1rem;
+    overflow-x: auto;
+}
+
+.code-block code {
+    background: transparent;
+    padding: 0;
+    color: var(--text-secondary);
+    font-size: 0.875rem;
+    line-height: 1.5;
+}
+
+/* Feature Grid */
+.feature-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+    gap: 1rem;
+    margin: 2rem 0;
+}
+
+.feature-card {
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    padding: 1.5rem;
+    transition: all 0.2s;
+}
+
+.feature-card:hover {
+    border-color: var(--primary-cyan);
+    background: rgba(80, 255, 255, 0.05);
+}
+
+.feature-card h4 {
+    margin-top: 0;
+}
+
+/* Info Box */
+.info-box {
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.05), rgba(243, 128, 245, 0.03));
+    border: 1px solid var(--primary-cyan);
+    border-left: 4px solid var(--primary-cyan);
+    padding: 1.5rem;
+    margin: 2rem 0;
+}
+
+.info-box h4 {
+    margin-top: 0;
+    color: var(--primary-cyan);
+}
+
+/* Support Grid */
+.support-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+    gap: 1rem;
+    margin: 2rem 0;
+}
+
+.support-card {
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    padding: 1.5rem;
+    text-align: center;
+}
+
+.support-card h3 {
+    color: var(--primary-cyan);
+    margin-bottom: 0.5rem;
+}
+
+/* Related Apps */
+.related-apps {
+    max-width: 1800px;
+    margin: 4rem auto;
+    padding: 0 2rem;
+}
+
+.related-apps h2 {
+    font-size: 1.5rem;
+    color: var(--text-primary);
+    margin-bottom: 1.5rem;
+}
+
+.related-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
+    gap: 1rem;
+}
+
+.related-app-card {
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    padding: 1rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.related-app-card:hover {
+    border-color: var(--primary-cyan);
+    transform: translateY(-2px);
+}
+
+/* Responsive */
+@media (max-width: 1024px) {
+    .app-hero-content {
+        grid-template-columns: 1fr;
+    }
+
+    .app-stats {
+        justify-content: space-around;
+    }
+}
+
+@media (max-width: 768px) {
+    .app-hero-info h1 {
+        font-size: 2rem;
+    }
+
+    .app-actions {
+        flex-direction: column;
+    }
+
+    .app-nav {
+        overflow-x: auto;
+        gap: 0;
+    }
+
+    .nav-tab {
+        white-space: nowrap;
+    }
+
+    .feature-grid,
+    .support-grid {
+        grid-template-columns: 1fr;
+    }
+}

docs/md_v2/marketplace/frontend/app-detail.html

@@ -0,0 +1,234 @@
+<!DOCTYPE html>
+<html lang="en" data-theme="dark">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>App Details - Crawl4AI Marketplace</title>
+    <link rel="stylesheet" href="marketplace.css">
+    <link rel="stylesheet" href="app-detail.css">
+</head>
+<body>
+    <div class="app-detail-container">
+        <!-- Header -->
+        <header class="marketplace-header">
+            <div class="header-content">
+                <div class="header-left">
+                    <div class="logo-title">
+                        <img src="../../assets/images/logo.png" alt="Crawl4AI" class="header-logo">
+                        <h1>
+                            <span class="ascii-border">[</span>
+                            Marketplace
+                            <span class="ascii-border">]</span>
+                        </h1>
+                    </div>
+                </div>
+                <div class="header-nav">
+                    <a href="index.html" class="back-btn">← Back to Marketplace</a>
+                </div>
+            </div>
+        </header>
+
+        <!-- App Hero Section -->
+        <section class="app-hero">
+            <div class="app-hero-content">
+                <div class="app-hero-image" id="app-image">
+                    <!-- Dynamic image -->
+                </div>
+                <div class="app-hero-info">
+                    <div class="app-badges">
+                        <span class="app-badge" id="app-type">Open Source</span>
+                        <span class="app-badge featured" id="app-featured" style="display:none">FEATURED</span>
+                        <span class="app-badge sponsored" id="app-sponsored" style="display:none">SPONSORED</span>
+                    </div>
+                    <h1 id="app-name">App Name</h1>
+                    <p id="app-description" class="app-tagline">App description goes here</p>
+
+                    <div class="app-stats">
+                        <div class="stat">
+                            <span class="stat-value" id="app-rating">★★★★★</span>
+                            <span class="stat-label">Rating</span>
+                        </div>
+                        <div class="stat">
+                            <span class="stat-value" id="app-downloads">0</span>
+                            <span class="stat-label">Downloads</span>
+                        </div>
+                        <div class="stat">
+                            <span class="stat-value" id="app-category">Category</span>
+                            <span class="stat-label">Category</span>
+                        </div>
+                    </div>
+
+                    <div class="app-actions">
+                        <a href="#" id="app-website" class="action-btn primary" target="_blank">
+                            <span>→</span> Visit Website
+                        </a>
+                        <a href="#" id="app-github" class="action-btn secondary" target="_blank">
+                            <span>⚡</span> View on GitHub
+                        </a>
+                        <button id="copy-integration" class="action-btn ghost">
+                            <span>📋</span> Copy Integration
+                        </button>
+                    </div>
+
+                    <div class="pricing-info">
+                        <span class="pricing-label">Pricing:</span>
+                        <span id="app-pricing" class="pricing-value">Free</span>
+                    </div>
+                </div>
+            </div>
+        </section>
+
+        <!-- Navigation Tabs -->
+        <nav class="app-nav">
+            <button class="nav-tab active" data-tab="integration">Integration Guide</button>
+            <button class="nav-tab" data-tab="docs">Documentation</button>
+            <button class="nav-tab" data-tab="examples">Examples</button>
+            <button class="nav-tab" data-tab="support">Support</button>
+        </nav>
+
+        <!-- Content Sections -->
+        <main class="app-content">
+            <!-- Integration Guide Tab -->
+            <section id="integration-tab" class="tab-content active">
+                <div class="docs-content">
+                    <h2>Quick Start</h2>
+                    <p>Get started with this integration in just a few steps.</p>
+
+                    <h3>Installation</h3>
+                    <div class="code-block">
+                        <div class="code-header">
+                            <span class="code-lang">bash</span>
+                            <button class="copy-btn">Copy</button>
+                        </div>
+                        <pre><code id="install-code">pip install crawl4ai</code></pre>
+                    </div>
+
+                    <h3>Basic Usage</h3>
+                    <div class="code-block">
+                        <div class="code-header">
+                            <span class="code-lang">python</span>
+                            <button class="copy-btn">Copy</button>
+                        </div>
+                        <pre><code id="usage-code">from crawl4ai import AsyncWebCrawler
+
+async def main():
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            # Your configuration here
+        )
+        print(result.markdown)
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())</code></pre>
+                    </div>
+
+                    <h3>Advanced Configuration</h3>
+                    <p>Customize the crawler with these advanced options:</p>
+
+                    <div class="feature-grid">
+                        <div class="feature-card">
+                            <h4>🚀 Performance</h4>
+                            <p>Optimize crawling speed with parallel processing and caching strategies.</p>
+                        </div>
+                        <div class="feature-card">
+                            <h4>🔒 Authentication</h4>
+                            <p>Handle login forms, cookies, and session management automatically.</p>
+                        </div>
+                        <div class="feature-card">
+                            <h4>🎯 Extraction</h4>
+                            <p>Use CSS selectors, XPath, or AI-powered content extraction.</p>
+                        </div>
+                        <div class="feature-card">
+                            <h4>🔄 Proxy Support</h4>
+                            <p>Rotate proxies and bypass rate limiting with built-in proxy management.</p>
+                        </div>
+                    </div>
+
+                    <h3>Integration Example</h3>
+                    <div class="code-block">
+                        <div class="code-header">
+                            <span class="code-lang">python</span>
+                            <button class="copy-btn">Copy</button>
+                        </div>
+                        <pre><code id="integration-code">from crawl4ai import AsyncWebCrawler
+from crawl4ai.extraction_strategy import LLMExtractionStrategy
+
+async def extract_with_llm():
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            extraction_strategy=LLMExtractionStrategy(
+                provider="openai",
+                api_key="your-api-key",
+                instruction="Extract product information"
+            ),
+            bypass_cache=True
+        )
+        return result.extracted_content
+
+# Run the extraction
+data = await extract_with_llm()
+print(data)</code></pre>
+                    </div>
+
+                    <div class="info-box">
+                        <h4>💡 Pro Tip</h4>
+                        <p>Use the <code>bypass_cache=True</code> parameter when you need fresh data, or set <code>cache_mode="write"</code> to update the cache with new content.</p>
+                    </div>
+                </div>
+            </section>
+
+            <!-- Documentation Tab -->
+            <section id="docs-tab" class="tab-content">
+                <div class="docs-content">
+                    <h2>Documentation</h2>
+                    <p>Complete documentation and API reference.</p>
+                    <!-- Dynamic content loaded here -->
+                </div>
+            </section>
+
+            <!-- Examples Tab -->
+            <section id="examples-tab" class="tab-content">
+                <div class="docs-content">
+                    <h2>Examples</h2>
+                    <p>Real-world examples and use cases.</p>
+                    <!-- Dynamic content loaded here -->
+                </div>
+            </section>
+
+            <!-- Support Tab -->
+            <section id="support-tab" class="tab-content">
+                <div class="docs-content">
+                    <h2>Support</h2>
+                    <div class="support-grid">
+                        <div class="support-card">
+                            <h3>📧 Contact</h3>
+                            <p id="app-contact">contact@example.com</p>
+                        </div>
+                        <div class="support-card">
+                            <h3>🐛 Report Issues</h3>
+                            <p>Found a bug? Report it on GitHub Issues.</p>
+                        </div>
+                        <div class="support-card">
+                            <h3>💬 Community</h3>
+                            <p>Join our Discord for help and discussions.</p>
+                        </div>
+                    </div>
+                </div>
+            </section>
+        </main>
+
+        <!-- Related Apps -->
+        <section class="related-apps">
+            <h2>Related Apps</h2>
+            <div id="related-apps-grid" class="related-grid">
+                <!-- Dynamic related apps -->
+            </div>
+        </section>
+    </div>
+
+    <script src="app-detail.js"></script>
+</body>
+</html>

docs/md_v2/marketplace/frontend/app-detail.js

@@ -0,0 +1,334 @@
+// App Detail Page JavaScript
+const { API_BASE, API_ORIGIN } = (() => {
+    const { hostname, port, protocol } = window.location;
+    const isLocalHost = ['localhost', '127.0.0.1', '0.0.0.0'].includes(hostname);
+
+    if (isLocalHost && port && port !== '8100') {
+        const origin = `${protocol}//127.0.0.1:8100`;
+        return { API_BASE: `${origin}/marketplace/api`, API_ORIGIN: origin };
+    }
+
+    return { API_BASE: '/marketplace/api', API_ORIGIN: '' };
+})();
+
+class AppDetailPage {
+    constructor() {
+        this.appSlug = this.getAppSlugFromURL();
+        this.appData = null;
+        this.init();
+    }
+
+    getAppSlugFromURL() {
+        const params = new URLSearchParams(window.location.search);
+        return params.get('app') || '';
+    }
+
+    async init() {
+        if (!this.appSlug) {
+            window.location.href = 'index.html';
+            return;
+        }
+
+        await this.loadAppDetails();
+        this.setupEventListeners();
+        await this.loadRelatedApps();
+    }
+
+    async loadAppDetails() {
+        try {
+            const response = await fetch(`${API_BASE}/apps/${this.appSlug}`);
+            if (!response.ok) throw new Error('App not found');
+
+            this.appData = await response.json();
+            this.renderAppDetails();
+        } catch (error) {
+            console.error('Error loading app details:', error);
+            // Fallback to loading all apps and finding the right one
+            try {
+                const response = await fetch(`${API_BASE}/apps`);
+                const apps = await response.json();
+                this.appData = apps.find(app => app.slug === this.appSlug || app.name.toLowerCase().replace(/\s+/g, '-') === this.appSlug);
+                if (this.appData) {
+                    this.renderAppDetails();
+                } else {
+                    window.location.href = 'index.html';
+                }
+            } catch (err) {
+                console.error('Error loading apps:', err);
+                window.location.href = 'index.html';
+            }
+        }
+    }
+
+    renderAppDetails() {
+        if (!this.appData) return;
+
+        // Update title
+        document.title = `${this.appData.name} - Crawl4AI Marketplace`;
+
+        // Hero image
+        const appImage = document.getElementById('app-image');
+        if (this.appData.image) {
+            appImage.style.backgroundImage = `url('${this.appData.image}')`;
+            appImage.innerHTML = '';
+        } else {
+            appImage.innerHTML = `[${this.appData.category || 'APP'}]`;
+        }
+
+        // Basic info
+        document.getElementById('app-name').textContent = this.appData.name;
+        document.getElementById('app-description').textContent = this.appData.description;
+        document.getElementById('app-type').textContent = this.appData.type || 'Open Source';
+        document.getElementById('app-category').textContent = this.appData.category;
+        document.getElementById('app-pricing').textContent = this.appData.pricing || 'Free';
+
+        // Badges
+        if (this.appData.featured) {
+            document.getElementById('app-featured').style.display = 'inline-block';
+        }
+        if (this.appData.sponsored) {
+            document.getElementById('app-sponsored').style.display = 'inline-block';
+        }
+
+        // Stats
+        const rating = this.appData.rating || 0;
+        const stars = '★'.repeat(Math.floor(rating)) + '☆'.repeat(5 - Math.floor(rating));
+        document.getElementById('app-rating').textContent = stars + ` ${rating}/5`;
+        document.getElementById('app-downloads').textContent = this.formatNumber(this.appData.downloads || 0);
+
+        // Action buttons
+        const websiteBtn = document.getElementById('app-website');
+        const githubBtn = document.getElementById('app-github');
+
+        if (this.appData.website_url) {
+            websiteBtn.href = this.appData.website_url;
+        } else {
+            websiteBtn.style.display = 'none';
+        }
+
+        if (this.appData.github_url) {
+            githubBtn.href = this.appData.github_url;
+        } else {
+            githubBtn.style.display = 'none';
+        }
+
+        // Contact
+        document.getElementById('app-contact').textContent = this.appData.contact_email || 'Not available';
+
+        // Integration guide
+        this.renderIntegrationGuide();
+    }
+
+    renderIntegrationGuide() {
+        // Installation code
+        const installCode = document.getElementById('install-code');
+        if (this.appData.type === 'Open Source' && this.appData.github_url) {
+            installCode.textContent = `# Clone from GitHub
+git clone ${this.appData.github_url}
+
+# Install dependencies
+pip install -r requirements.txt`;
+        } else if (this.appData.name.toLowerCase().includes('api')) {
+            installCode.textContent = `# Install via pip
+pip install ${this.appData.slug}
+
+# Or install from source
+pip install git+${this.appData.github_url || 'https://github.com/example/repo'}`;
+        }
+
+        // Usage code - customize based on category
+        const usageCode = document.getElementById('usage-code');
+        if (this.appData.category === 'Browser Automation') {
+            usageCode.textContent = `from crawl4ai import AsyncWebCrawler
+from ${this.appData.slug.replace(/-/g, '_')} import ${this.appData.name.replace(/\s+/g, '')}
+
+async def main():
+    # Initialize ${this.appData.name}
+    automation = ${this.appData.name.replace(/\s+/g, '')}()
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            browser_config=automation.config,
+            wait_for="css:body"
+        )
+        print(result.markdown)`;
+        } else if (this.appData.category === 'Proxy Services') {
+            usageCode.textContent = `from crawl4ai import AsyncWebCrawler
+import ${this.appData.slug.replace(/-/g, '_')}
+
+# Configure proxy
+proxy_config = {
+    "server": "${this.appData.website_url || 'https://proxy.example.com'}",
+    "username": "your_username",
+    "password": "your_password"
+}
+
+async with AsyncWebCrawler(proxy=proxy_config) as crawler:
+    result = await crawler.arun(
+        url="https://example.com",
+        bypass_cache=True
+    )
+    print(result.status_code)`;
+        } else if (this.appData.category === 'LLM Integration') {
+            usageCode.textContent = `from crawl4ai import AsyncWebCrawler
+from crawl4ai.extraction_strategy import LLMExtractionStrategy
+
+# Configure LLM extraction
+strategy = LLMExtractionStrategy(
+    provider="${this.appData.name.toLowerCase().includes('gpt') ? 'openai' : 'anthropic'}",
+    api_key="your-api-key",
+    model="${this.appData.name.toLowerCase().includes('gpt') ? 'gpt-4' : 'claude-3'}",
+    instruction="Extract structured data"
+)
+
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun(
+        url="https://example.com",
+        extraction_strategy=strategy
+    )
+    print(result.extracted_content)`;
+        }
+
+        // Integration example
+        const integrationCode = document.getElementById('integration-code');
+        integrationCode.textContent = this.appData.integration_guide ||
+`# Complete ${this.appData.name} Integration Example
+
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+import json
+
+async def crawl_with_${this.appData.slug.replace(/-/g, '_')}():
+    """
+    Complete example showing how to use ${this.appData.name}
+    with Crawl4AI for production web scraping
+    """
+
+    # Define extraction schema
+    schema = {
+        "name": "ProductList",
+        "baseSelector": "div.product",
+        "fields": [
+            {"name": "title", "selector": "h2", "type": "text"},
+            {"name": "price", "selector": ".price", "type": "text"},
+            {"name": "image", "selector": "img", "type": "attribute", "attribute": "src"},
+            {"name": "link", "selector": "a", "type": "attribute", "attribute": "href"}
+        ]
+    }
+
+    # Initialize crawler with ${this.appData.name}
+    async with AsyncWebCrawler(
+        browser_type="chromium",
+        headless=True,
+        verbose=True
+    ) as crawler:
+
+        # Crawl with extraction
+        result = await crawler.arun(
+            url="https://example.com/products",
+            extraction_strategy=JsonCssExtractionStrategy(schema),
+            cache_mode="bypass",
+            wait_for="css:.product",
+            screenshot=True
+        )
+
+        # Process results
+        if result.success:
+            products = json.loads(result.extracted_content)
+            print(f"Found {len(products)} products")
+
+            for product in products[:5]:
+                print(f"- {product['title']}: {product['price']}")
+
+        return products
+
+# Run the crawler
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(crawl_with_${this.appData.slug.replace(/-/g, '_')}())`;
+    }
+
+    formatNumber(num) {
+        if (num >= 1000000) {
+            return (num / 1000000).toFixed(1) + 'M';
+        } else if (num >= 1000) {
+            return (num / 1000).toFixed(1) + 'K';
+        }
+        return num.toString();
+    }
+
+    setupEventListeners() {
+        // Tab switching
+        const tabs = document.querySelectorAll('.nav-tab');
+        tabs.forEach(tab => {
+            tab.addEventListener('click', () => {
+                // Update active tab
+                tabs.forEach(t => t.classList.remove('active'));
+                tab.classList.add('active');
+
+                // Show corresponding content
+                const tabName = tab.dataset.tab;
+                document.querySelectorAll('.tab-content').forEach(content => {
+                    content.classList.remove('active');
+                });
+                document.getElementById(`${tabName}-tab`).classList.add('active');
+            });
+        });
+
+        // Copy integration code
+        document.getElementById('copy-integration').addEventListener('click', () => {
+            const code = document.getElementById('integration-code').textContent;
+            navigator.clipboard.writeText(code).then(() => {
+                const btn = document.getElementById('copy-integration');
+                const originalText = btn.innerHTML;
+                btn.innerHTML = '<span>✓</span> Copied!';
+                setTimeout(() => {
+                    btn.innerHTML = originalText;
+                }, 2000);
+            });
+        });
+
+        // Copy code buttons
+        document.querySelectorAll('.copy-btn').forEach(btn => {
+            btn.addEventListener('click', (e) => {
+                const codeBlock = e.target.closest('.code-block');
+                const code = codeBlock.querySelector('code').textContent;
+                navigator.clipboard.writeText(code).then(() => {
+                    btn.textContent = 'Copied!';
+                    setTimeout(() => {
+                        btn.textContent = 'Copy';
+                    }, 2000);
+                });
+            });
+        });
+    }
+
+    async loadRelatedApps() {
+        try {
+            const response = await fetch(`${API_BASE}/apps?category=${encodeURIComponent(this.appData.category)}&limit=4`);
+            const apps = await response.json();
+
+            const relatedApps = apps.filter(app => app.slug !== this.appSlug).slice(0, 3);
+            const grid = document.getElementById('related-apps-grid');
+
+            grid.innerHTML = relatedApps.map(app => `
+                <div class="related-app-card" onclick="window.location.href='app-detail.html?app=${app.slug || app.name.toLowerCase().replace(/\s+/g, '-')}'">
+                    <h4>${app.name}</h4>
+                    <p>${app.description.substring(0, 100)}...</p>
+                    <div style="display: flex; justify-content: space-between; margin-top: 0.5rem; font-size: 0.75rem;">
+                        <span style="color: var(--primary-cyan)">${app.type}</span>
+                        <span style="color: var(--warning)">★ ${app.rating}/5</span>
+                    </div>
+                </div>
+            `).join('');
+        } catch (error) {
+            console.error('Error loading related apps:', error);
+        }
+    }
+}
+
+// Initialize when DOM is loaded
+document.addEventListener('DOMContentLoaded', () => {
+    new AppDetailPage();
+});

docs/md_v2/marketplace/frontend/index.html

@@ -0,0 +1,147 @@
+<!DOCTYPE html>
+<html lang="en" data-theme="dark">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Marketplace - Crawl4AI</title>
+    <link rel="stylesheet" href="marketplace.css">
+</head>
+<body>
+    <div class="marketplace-container">
+        <!-- Header -->
+        <header class="marketplace-header">
+            <div class="header-content">
+                <div class="header-left">
+                    <div class="logo-title">
+                        <img src="../../assets/images/logo.png" alt="Crawl4AI" class="header-logo">
+                        <h1>
+                            <span class="ascii-border">[</span>
+                            Marketplace
+                            <span class="ascii-border">]</span>
+                        </h1>
+                    </div>
+                    <p class="tagline">Tools, Integrations & Resources for Web Crawling</p>
+                </div>
+                <div class="header-stats" id="stats">
+                    <span class="stat-item">Apps: <span id="total-apps">--</span></span>
+                    <span class="stat-item">Articles: <span id="total-articles">--</span></span>
+                    <span class="stat-item">Downloads: <span id="total-downloads">--</span></span>
+                </div>
+            </div>
+        </header>
+
+        <!-- Search and Category Bar -->
+        <div class="search-filter-bar">
+            <div class="search-box">
+                <span class="search-icon">></span>
+                <input type="text" id="search-input" placeholder="Search apps, articles, tools..." />
+                <kbd>/</kbd>
+            </div>
+            <div class="category-filter" id="category-filter">
+                <button class="filter-btn active" data-category="all">All</button>
+                <!-- Categories will be loaded here -->
+            </div>
+        </div>
+
+        <!-- Magazine Grid Layout -->
+        <main class="magazine-layout">
+            <!-- Hero Featured Section -->
+            <section class="hero-featured">
+                <div id="featured-hero" class="featured-hero-card">
+                    <!-- Large featured card with big image -->
+                </div>
+            </section>
+
+            <!-- Secondary Featured -->
+            <section class="secondary-featured">
+                <div id="featured-secondary" class="featured-secondary-cards">
+                    <!-- 2-3 medium featured cards with images -->
+                </div>
+            </section>
+
+            <!-- Sponsored Section -->
+            <section class="sponsored-section">
+                <div class="section-label">SPONSORED</div>
+                <div id="sponsored-content" class="sponsored-cards">
+                    <!-- Sponsored content cards -->
+                </div>
+            </section>
+
+            <!-- Main Content Grid -->
+            <section class="main-content">
+                <!-- Apps Column -->
+                <div class="apps-column">
+                    <div class="column-header">
+                        <h2><span class="ascii-icon">></span> Latest Apps</h2>
+                        <select id="type-filter" class="mini-filter">
+                            <option value="">All</option>
+                            <option value="Open Source">Open Source</option>
+                            <option value="Paid">Paid</option>
+                        </select>
+                    </div>
+                    <div id="apps-grid" class="apps-compact-grid">
+                        <!-- Compact app cards -->
+                    </div>
+                </div>
+
+                <!-- Articles Column -->
+                <div class="articles-column">
+                    <div class="column-header">
+                        <h2><span class="ascii-icon">></span> Latest Articles</h2>
+                    </div>
+                    <div id="articles-list" class="articles-compact-list">
+                        <!-- Article items -->
+                    </div>
+                </div>
+
+                <!-- Trending/Tools Column -->
+                <div class="trending-column">
+                    <div class="column-header">
+                        <h2><span class="ascii-icon">#</span> Trending</h2>
+                    </div>
+                    <div id="trending-list" class="trending-items">
+                        <!-- Trending items -->
+                    </div>
+
+                    <div class="submit-box">
+                        <h3><span class="ascii-icon">+</span> Submit Your Tool</h3>
+                        <p>Share your integration</p>
+                        <a href="mailto:marketplace@crawl4ai.com" class="submit-btn">Submit →</a>
+                    </div>
+                </div>
+            </section>
+
+            <!-- More Apps Grid -->
+            <section class="more-apps">
+                <div class="section-header">
+                    <h2><span class="ascii-icon">></span> More Apps</h2>
+                    <button id="load-more" class="load-more-btn">Load More ↓</button>
+                </div>
+                <div id="more-apps-grid" class="more-apps-grid">
+                    <!-- Additional app cards -->
+                </div>
+            </section>
+        </main>
+
+        <!-- Footer -->
+        <footer class="marketplace-footer">
+            <div class="footer-content">
+                <div class="footer-section">
+                    <h3>About Marketplace</h3>
+                    <p>Discover tools and integrations built by the Crawl4AI community.</p>
+                </div>
+                <div class="footer-section">
+                    <h3>Become a Sponsor</h3>
+                    <p>Reach developers building with Crawl4AI</p>
+                    <a href="mailto:sponsors@crawl4ai.com" class="sponsor-btn">Learn More →</a>
+                </div>
+            </div>
+            <div class="footer-bottom">
+                <p>[ Crawl4AI Marketplace · Updated <span id="last-update">--</span> ]</p>
+            </div>
+        </footer>
+    </div>
+
+    <script src="marketplace.js"></script>
+</body>
+</html>

docs/md_v2/marketplace/frontend/marketplace.css

@@ -0,0 +1,957 @@
+/* Marketplace CSS - Magazine Style Terminal Theme */
+@import url('../../assets/styles.css');
+
+:root {
+    --primary-cyan: #50ffff;
+    --primary-teal: #09b5a5;
+    --accent-pink: #f380f5;
+    --bg-dark: #070708;
+    --bg-secondary: #1a1a1a;
+    --bg-tertiary: #3f3f44;
+    --text-primary: #e8e9ed;
+    --text-secondary: #d5cec0;
+    --text-tertiary: #a3abba;
+    --border-color: #3f3f44;
+    --success: #50ff50;
+    --error: #ff3c74;
+    --warning: #f59e0b;
+}
+
+* {
+    margin: 0;
+    padding: 0;
+    box-sizing: border-box;
+}
+
+body {
+    font-family: 'Dank Mono', Monaco, monospace;
+    background: var(--bg-dark);
+    color: var(--text-primary);
+    line-height: 1.6;
+}
+
+/* Global link styles */
+a {
+    color: var(--primary-cyan);
+    text-decoration: none;
+    transition: color 0.2s;
+}
+
+a:hover {
+    color: var(--accent-pink);
+}
+
+.marketplace-container {
+    min-height: 100vh;
+}
+
+/* Header */
+.marketplace-header {
+    background: var(--bg-secondary);
+    border-bottom: 1px solid var(--border-color);
+    padding: 1.5rem 0;
+}
+
+.header-content {
+    max-width: 1800px;
+    margin: 0 auto;
+    padding: 0 2rem;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+}
+
+.logo-title {
+    display: flex;
+    align-items: center;
+    gap: 1rem;
+}
+
+.header-logo {
+    height: 40px;
+    width: auto;
+    filter: brightness(1.2);
+}
+
+.marketplace-header h1 {
+    font-size: 1.5rem;
+    color: var(--primary-cyan);
+    margin: 0;
+}
+
+.ascii-border {
+    color: var(--border-color);
+}
+
+.tagline {
+    font-size: 0.875rem;
+    color: var(--text-tertiary);
+    margin-top: 0.25rem;
+}
+
+.header-stats {
+    display: flex;
+    gap: 2rem;
+}
+
+.stat-item {
+    font-size: 0.875rem;
+    color: var(--text-secondary);
+}
+
+.stat-item span {
+    color: var(--primary-cyan);
+    font-weight: 600;
+}
+
+/* Search and Filter Bar */
+.search-filter-bar {
+    max-width: 1800px;
+    margin: 1.5rem auto;
+    padding: 0 2rem;
+    display: flex;
+    gap: 1rem;
+    align-items: center;
+}
+
+.search-box {
+    flex: 1;
+    max-width: 500px;
+    display: flex;
+    align-items: center;
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    padding: 0.75rem 1rem;
+    transition: border-color 0.2s;
+}
+
+.search-box:focus-within {
+    border-color: var(--primary-cyan);
+}
+
+.search-icon {
+    color: var(--text-tertiary);
+    margin-right: 1rem;
+}
+
+#search-input {
+    flex: 1;
+    background: transparent;
+    border: none;
+    color: var(--text-primary);
+    font-family: inherit;
+    font-size: 0.9rem;
+    outline: none;
+}
+
+.search-box kbd {
+    font-size: 0.75rem;
+    padding: 0.2rem 0.5rem;
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    color: var(--text-tertiary);
+}
+
+.category-filter {
+    display: flex;
+    gap: 0.5rem;
+    flex-wrap: wrap;
+}
+
+.filter-btn {
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    padding: 0.5rem 1rem;
+    font-family: inherit;
+    font-size: 0.875rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.filter-btn:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+.filter-btn.active {
+    background: var(--primary-cyan);
+    color: var(--bg-dark);
+    border-color: var(--primary-cyan);
+}
+
+/* Magazine Layout */
+.magazine-layout {
+    max-width: 1800px;
+    margin: 0 auto;
+    padding: 0 2rem 4rem;
+    display: grid;
+    grid-template-columns: 1fr;
+    gap: 2rem;
+}
+
+/* Hero Featured Section */
+.hero-featured {
+    grid-column: 1 / -1;
+    position: relative;
+}
+
+.hero-featured::before {
+    content: '';
+    position: absolute;
+    top: -20px;
+    left: -20px;
+    right: -20px;
+    bottom: -20px;
+    background: radial-gradient(ellipse at center, rgba(80, 255, 255, 0.05), transparent 70%);
+    pointer-events: none;
+    z-index: -1;
+}
+
+.featured-hero-card {
+    background: linear-gradient(135deg, #1a1a2e, #0f0f1e);
+    border: 2px solid var(--primary-cyan);
+    box-shadow: 0 0 30px rgba(80, 255, 255, 0.15),
+                inset 0 0 20px rgba(80, 255, 255, 0.05);
+    height: 380px;
+    position: relative;
+    overflow: hidden;
+    cursor: pointer;
+    transition: all 0.3s ease;
+    display: flex;
+    flex-direction: column;
+}
+
+.featured-hero-card:hover {
+    border-color: var(--accent-pink);
+    box-shadow: 0 0 40px rgba(243, 128, 245, 0.2),
+                inset 0 0 30px rgba(243, 128, 245, 0.05);
+    transform: translateY(-2px);
+}
+
+.hero-image {
+    width: 100%;
+    height: 240px;
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.1), rgba(243, 128, 245, 0.05));
+    background-size: cover;
+    background-position: center;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-size: 3rem;
+    color: var(--primary-cyan);
+    flex-shrink: 0;
+    position: relative;
+    filter: brightness(1.1) contrast(1.1);
+}
+
+.hero-image::after {
+    content: '';
+    position: absolute;
+    bottom: 0;
+    left: 0;
+    right: 0;
+    height: 60%;
+    background: linear-gradient(to top, rgba(10, 10, 20, 0.95), transparent);
+}
+
+.hero-content {
+    padding: 1.5rem;
+}
+
+.hero-badge {
+    display: inline-block;
+    padding: 0.3rem 0.6rem;
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    color: var(--bg-dark);
+    font-size: 0.7rem;
+    text-transform: uppercase;
+    margin-bottom: 0.5rem;
+    font-weight: 600;
+    box-shadow: 0 2px 10px rgba(80, 255, 255, 0.3);
+}
+
+.hero-title {
+    font-size: 1.6rem;
+    color: var(--primary-cyan);
+    margin: 0.5rem 0;
+    text-shadow: 0 0 20px rgba(80, 255, 255, 0.5);
+}
+
+.hero-description {
+    color: var(--text-secondary);
+    line-height: 1.5;
+}
+
+.hero-meta {
+    display: flex;
+    gap: 1.5rem;
+    margin-top: 1rem;
+    font-size: 0.875rem;
+}
+
+.hero-meta span {
+    color: var(--text-tertiary);
+}
+
+.hero-meta span:first-child {
+    color: var(--warning);
+}
+
+/* Secondary Featured */
+.secondary-featured {
+    grid-column: 1 / -1;
+    height: 380px;
+    display: flex;
+    align-items: stretch;
+}
+
+.featured-secondary-cards {
+    width: 100%;
+    display: flex;
+    flex-direction: column;
+    gap: 0.75rem;
+    justify-content: space-between;
+}
+
+.secondary-card {
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.03), rgba(243, 128, 245, 0.02));
+    border: 1px solid rgba(80, 255, 255, 0.3);
+    cursor: pointer;
+    transition: all 0.3s ease;
+    display: flex;
+    overflow: hidden;
+    height: calc((380px - 1.5rem) / 3);
+    flex: 1;
+    box-shadow: 0 2px 10px rgba(0, 0, 0, 0.3);
+}
+
+.secondary-card:hover {
+    border-color: var(--accent-pink);
+    background: linear-gradient(135deg, rgba(243, 128, 245, 0.05), rgba(80, 255, 255, 0.03));
+    box-shadow: 0 4px 15px rgba(243, 128, 245, 0.2);
+    transform: translateX(-3px);
+}
+
+.secondary-image {
+    width: 120px;
+    background: linear-gradient(135deg, var(--bg-tertiary), var(--bg-secondary));
+    background-size: cover;
+    background-position: center;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-size: 1.5rem;
+    color: var(--primary-cyan);
+    flex-shrink: 0;
+}
+
+.secondary-content {
+    flex: 1;
+    padding: 1rem;
+    display: flex;
+    flex-direction: column;
+    justify-content: space-between;
+}
+
+.secondary-title {
+    font-size: 1rem;
+    color: var(--text-primary);
+    margin-bottom: 0.25rem;
+}
+
+.secondary-desc {
+    font-size: 0.75rem;
+    color: var(--text-secondary);
+    display: -webkit-box;
+    -webkit-line-clamp: 2;
+    -webkit-box-orient: vertical;
+    overflow: hidden;
+}
+
+.secondary-meta {
+    font-size: 0.75rem;
+    color: var(--text-tertiary);
+}
+
+.secondary-meta span:last-child {
+    color: var(--warning);
+}
+
+/* Sponsored Section */
+.sponsored-section {
+    grid-column: 1 / -1;
+    background: var(--bg-secondary);
+    border: 1px solid var(--warning);
+    padding: 1rem;
+    position: relative;
+}
+
+.section-label {
+    position: absolute;
+    top: -0.5rem;
+    left: 1rem;
+    background: var(--bg-secondary);
+    padding: 0 0.5rem;
+    color: var(--warning);
+    font-size: 0.65rem;
+    letter-spacing: 0.1em;
+}
+
+.sponsored-cards {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+    gap: 1rem;
+}
+
+.sponsor-card {
+    padding: 1rem;
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+}
+
+.sponsor-card h4 {
+    color: var(--accent-pink);
+    margin-bottom: 0.5rem;
+}
+
+.sponsor-card p {
+    color: var(--text-secondary);
+    font-size: 0.85rem;
+    margin-bottom: 0.75rem;
+}
+
+.sponsor-card a {
+    color: var(--primary-cyan);
+    text-decoration: none;
+    font-size: 0.85rem;
+}
+
+.sponsor-card a:hover {
+    color: var(--accent-pink);
+}
+
+/* Main Content Grid */
+.main-content {
+    grid-column: 1 / -1;
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
+    gap: 2rem;
+}
+
+/* Column Headers */
+.column-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 1rem;
+    border-bottom: 1px solid var(--border-color);
+    padding-bottom: 0.5rem;
+}
+
+.column-header h2 {
+    font-size: 1.1rem;
+    color: var(--text-primary);
+}
+
+.mini-filter {
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    color: var(--text-primary);
+    padding: 0.25rem 0.5rem;
+    font-family: inherit;
+    font-size: 0.75rem;
+}
+
+.ascii-icon {
+    color: var(--primary-cyan);
+}
+
+/* Apps Column */
+.apps-compact-grid {
+    display: flex;
+    flex-direction: column;
+    gap: 0.75rem;
+}
+
+.app-compact {
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    border-left: 3px solid var(--border-color);
+    padding: 0.75rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.app-compact:hover {
+    border-color: var(--primary-cyan);
+    border-left-color: var(--accent-pink);
+    transform: translateX(2px);
+}
+
+.app-compact-header {
+    display: flex;
+    justify-content: space-between;
+    font-size: 0.75rem;
+    color: var(--text-tertiary);
+    margin-bottom: 0.25rem;
+}
+
+.app-compact-header span:first-child {
+    color: var(--primary-cyan);
+}
+
+.app-compact-header span:last-child {
+    color: var(--warning);
+}
+
+.app-compact-title {
+    font-size: 0.9rem;
+    color: var(--text-primary);
+    margin-bottom: 0.25rem;
+}
+
+.app-compact-desc {
+    font-size: 0.75rem;
+    color: var(--text-secondary);
+    display: -webkit-box;
+    -webkit-line-clamp: 2;
+    -webkit-box-orient: vertical;
+    overflow: hidden;
+}
+
+/* Articles Column */
+.articles-compact-list {
+    display: flex;
+    flex-direction: column;
+    gap: 1rem;
+}
+
+.article-compact {
+    border-left: 2px solid var(--border-color);
+    padding-left: 1rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.article-compact:hover {
+    border-left-color: var(--primary-cyan);
+}
+
+.article-meta {
+    font-size: 0.7rem;
+    color: var(--text-tertiary);
+    margin-bottom: 0.25rem;
+}
+
+.article-meta span:first-child {
+    color: var(--accent-pink);
+}
+
+.article-title {
+    font-size: 0.9rem;
+    color: var(--text-primary);
+    margin-bottom: 0.25rem;
+}
+
+.article-author {
+    font-size: 0.75rem;
+    color: var(--text-secondary);
+}
+
+/* Trending Column */
+.trending-items {
+    display: flex;
+    flex-direction: column;
+    gap: 0.5rem;
+}
+
+.trending-item {
+    display: flex;
+    align-items: center;
+    gap: 0.75rem;
+    padding: 0.5rem;
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.trending-item:hover {
+    border-color: var(--primary-cyan);
+}
+
+.trending-rank {
+    font-size: 1.2rem;
+    color: var(--primary-cyan);
+    width: 2rem;
+    text-align: center;
+}
+
+.trending-info {
+    flex: 1;
+}
+
+.trending-name {
+    font-size: 0.85rem;
+    color: var(--text-primary);
+}
+
+.trending-stats {
+    font-size: 0.7rem;
+    color: var(--text-tertiary);
+}
+
+/* Submit Box */
+.submit-box {
+    margin-top: 1.5rem;
+    background: var(--bg-secondary);
+    border: 1px solid var(--primary-cyan);
+    padding: 1rem;
+    text-align: center;
+}
+
+.submit-box h3 {
+    font-size: 1rem;
+    color: var(--primary-cyan);
+    margin-bottom: 0.5rem;
+}
+
+.submit-box p {
+    font-size: 0.8rem;
+    color: var(--text-secondary);
+    margin-bottom: 0.75rem;
+}
+
+.submit-btn {
+    display: inline-block;
+    padding: 0.5rem 1rem;
+    background: transparent;
+    border: 1px solid var(--primary-cyan);
+    color: var(--primary-cyan);
+    text-decoration: none;
+    transition: all 0.2s;
+}
+
+.submit-btn:hover {
+    background: var(--primary-cyan);
+    color: var(--bg-dark);
+}
+
+/* More Apps Section */
+.more-apps {
+    grid-column: 1 / -1;
+    margin-top: 2rem;
+}
+
+.section-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 1rem;
+}
+
+.more-apps-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
+    gap: 1rem;
+}
+
+.load-more-btn {
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    padding: 0.5rem 1.5rem;
+    font-family: inherit;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.load-more-btn:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+/* Footer */
+.marketplace-footer {
+    background: var(--bg-secondary);
+    border-top: 1px solid var(--border-color);
+    margin-top: 4rem;
+    padding: 2rem 0;
+}
+
+.footer-content {
+    max-width: 1800px;
+    margin: 0 auto;
+    padding: 0 2rem;
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    gap: 2rem;
+}
+
+.footer-section h3 {
+    font-size: 1rem;
+    margin-bottom: 0.5rem;
+    color: var(--primary-cyan);
+}
+
+.footer-section p {
+    font-size: 0.875rem;
+    color: var(--text-secondary);
+    margin-bottom: 1rem;
+}
+
+.sponsor-btn {
+    display: inline-block;
+    padding: 0.5rem 1rem;
+    background: transparent;
+    border: 1px solid var(--primary-cyan);
+    color: var(--primary-cyan);
+    text-decoration: none;
+    transition: all 0.2s;
+}
+
+.sponsor-btn:hover {
+    background: var(--primary-cyan);
+    color: var(--bg-dark);
+}
+
+.footer-bottom {
+    max-width: 1800px;
+    margin: 2rem auto 0;
+    padding: 1rem 2rem 0;
+    border-top: 1px solid var(--border-color);
+    font-size: 0.75rem;
+    color: var(--text-tertiary);
+}
+
+/* Modal */
+.modal {
+    position: fixed;
+    top: 0;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    background: rgba(0, 0, 0, 0.8);
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    z-index: 1000;
+}
+
+.modal.hidden {
+    display: none;
+}
+
+.modal-content {
+    background: var(--bg-secondary);
+    border: 1px solid var(--primary-cyan);
+    max-width: 800px;
+    width: 90%;
+    max-height: 80vh;
+    overflow-y: auto;
+    position: relative;
+}
+
+.modal-close {
+    position: absolute;
+    top: 1rem;
+    right: 1rem;
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-primary);
+    padding: 0.25rem 0.5rem;
+    cursor: pointer;
+    font-size: 1.2rem;
+}
+
+.modal-close:hover {
+    border-color: var(--error);
+    color: var(--error);
+}
+
+.app-detail {
+    padding: 2rem;
+}
+
+.app-detail h2 {
+    font-size: 1.5rem;
+    margin-bottom: 1rem;
+    color: var(--primary-cyan);
+}
+
+/* Loading */
+.loading {
+    text-align: center;
+    padding: 2rem;
+    color: var(--text-tertiary);
+}
+
+.no-results {
+    text-align: center;
+    padding: 2rem;
+    color: var(--text-tertiary);
+}
+
+/* Responsive - Tablet */
+@media (min-width: 768px) {
+    .magazine-layout {
+        grid-template-columns: repeat(2, 1fr);
+    }
+
+    .hero-featured {
+        grid-column: 1 / -1;
+    }
+
+    .secondary-featured {
+        grid-column: 1 / -1;
+    }
+
+    .sponsored-section {
+        grid-column: 1 / -1;
+    }
+
+    .main-content {
+        grid-column: 1 / -1;
+        grid-template-columns: repeat(2, 1fr);
+    }
+}
+
+/* Responsive - Desktop */
+@media (min-width: 1024px) {
+    .magazine-layout {
+        grid-template-columns: repeat(3, 1fr);
+    }
+
+    .hero-featured {
+        grid-column: 1 / 3;
+        grid-row: 1;
+    }
+
+    .secondary-featured {
+        grid-column: 3 / 4;
+        grid-row: 1;
+    }
+
+    .featured-secondary-cards {
+        flex-direction: column;
+    }
+
+    .sponsored-section {
+        grid-column: 1 / -1;
+    }
+
+    .main-content {
+        grid-column: 1 / -1;
+        grid-template-columns: repeat(3, 1fr);
+    }
+}
+
+/* Responsive - Wide Desktop */
+@media (min-width: 1400px) {
+    .magazine-layout {
+        grid-template-columns: repeat(4, 1fr);
+    }
+
+    .hero-featured {
+        grid-column: 1 / 3;
+    }
+
+    .secondary-featured {
+        grid-column: 3 / 5;
+        grid-row: 1;
+    }
+
+    .featured-secondary-cards {
+        grid-template-columns: repeat(2, 1fr);
+    }
+
+    .main-content {
+        grid-template-columns: repeat(4, 1fr);
+    }
+
+    .apps-column {
+        grid-column: span 2;
+    }
+
+    .more-apps-grid {
+        grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
+    }
+}
+
+/* Responsive - Ultra Wide Desktop (for coders with wide monitors) */
+@media (min-width: 1800px) {
+    .magazine-layout {
+        grid-template-columns: repeat(5, 1fr);
+    }
+
+    .hero-featured {
+        grid-column: 1 / 3;
+    }
+
+    .secondary-featured {
+        grid-column: 3 / 6;
+    }
+
+    .featured-secondary-cards {
+        grid-template-columns: repeat(3, 1fr);
+    }
+
+    .sponsored-section {
+        grid-column: 1 / -1;
+    }
+
+    .sponsored-cards {
+        grid-template-columns: repeat(5, 1fr);
+    }
+
+    .main-content {
+        grid-template-columns: repeat(5, 1fr);
+    }
+
+    .apps-column {
+        grid-column: span 2;
+    }
+
+    .articles-column {
+        grid-column: span 2;
+    }
+
+    .more-apps-grid {
+        grid-template-columns: repeat(auto-fill, minmax(300px, 1fr));
+    }
+}
+
+/* Responsive - Mobile */
+@media (max-width: 767px) {
+    .header-content {
+        flex-direction: column;
+        gap: 1rem;
+    }
+
+    .search-filter-bar {
+        flex-direction: column;
+        align-items: stretch;
+    }
+
+    .search-box {
+        max-width: none;
+    }
+
+    .magazine-layout {
+        padding: 0 1rem 2rem;
+    }
+
+    .footer-content {
+        grid-template-columns: 1fr;
+    }
+
+    .secondary-card {
+        flex-direction: column;
+    }
+
+    .secondary-image {
+        width: 100%;
+        height: 150px;
+    }
+}

docs/md_v2/marketplace/frontend/marketplace.js

@@ -0,0 +1,395 @@
+// Marketplace JS - Magazine Layout
+const API_BASE = '/marketplace/api';
+const CACHE_TTL = 3600000; // 1 hour in ms
+
+class MarketplaceCache {
+    constructor() {
+        this.prefix = 'c4ai_market_';
+    }
+
+    get(key) {
+        const item = localStorage.getItem(this.prefix + key);
+        if (!item) return null;
+
+        const data = JSON.parse(item);
+        if (Date.now() > data.expires) {
+            localStorage.removeItem(this.prefix + key);
+            return null;
+        }
+        return data.value;
+    }
+
+    set(key, value, ttl = CACHE_TTL) {
+        const data = {
+            value: value,
+            expires: Date.now() + ttl
+        };
+        localStorage.setItem(this.prefix + key, JSON.stringify(data));
+    }
+
+    clear() {
+        Object.keys(localStorage)
+            .filter(k => k.startsWith(this.prefix))
+            .forEach(k => localStorage.removeItem(k));
+    }
+}
+
+class MarketplaceAPI {
+    constructor() {
+        this.cache = new MarketplaceCache();
+        this.searchTimeout = null;
+    }
+
+    async fetch(endpoint, useCache = true) {
+        const cacheKey = endpoint.replace(/[^\w]/g, '_');
+
+        if (useCache) {
+            const cached = this.cache.get(cacheKey);
+            if (cached) return cached;
+        }
+
+        try {
+            const response = await fetch(`${API_BASE}${endpoint}`);
+            if (!response.ok) throw new Error(`HTTP ${response.status}`);
+
+            const data = await response.json();
+            this.cache.set(cacheKey, data);
+            return data;
+        } catch (error) {
+            console.error('API Error:', error);
+            return null;
+        }
+    }
+
+    async getStats() {
+        return this.fetch('/stats');
+    }
+
+    async getCategories() {
+        return this.fetch('/categories');
+    }
+
+    async getApps(params = {}) {
+        const query = new URLSearchParams(params).toString();
+        return this.fetch(`/apps${query ? '?' + query : ''}`);
+    }
+
+    async getArticles(params = {}) {
+        const query = new URLSearchParams(params).toString();
+        return this.fetch(`/articles${query ? '?' + query : ''}`);
+    }
+
+    async getSponsors() {
+        return this.fetch('/sponsors');
+    }
+
+    async search(query) {
+        if (query.length < 2) return {};
+        return this.fetch(`/search?q=${encodeURIComponent(query)}`, false);
+    }
+}
+
+class MarketplaceUI {
+    constructor() {
+        this.api = new MarketplaceAPI();
+        this.currentCategory = 'all';
+        this.currentType = '';
+        this.searchTimeout = null;
+        this.loadedApps = 10;
+        this.init();
+    }
+
+    async init() {
+        await this.loadStats();
+        await this.loadCategories();
+        await this.loadFeaturedContent();
+        await this.loadSponsors();
+        await this.loadMainContent();
+        this.setupEventListeners();
+    }
+
+    async loadStats() {
+        const stats = await this.api.getStats();
+        if (stats) {
+            document.getElementById('total-apps').textContent = stats.total_apps || '0';
+            document.getElementById('total-articles').textContent = stats.total_articles || '0';
+            document.getElementById('total-downloads').textContent = stats.total_downloads || '0';
+            document.getElementById('last-update').textContent = new Date().toLocaleDateString();
+        }
+    }
+
+    async loadCategories() {
+        const categories = await this.api.getCategories();
+        if (!categories) return;
+
+        const filter = document.getElementById('category-filter');
+        categories.forEach(cat => {
+            const btn = document.createElement('button');
+            btn.className = 'filter-btn';
+            btn.dataset.category = cat.slug;
+            btn.textContent = cat.name;
+            btn.onclick = () => this.filterByCategory(cat.slug);
+            filter.appendChild(btn);
+        });
+    }
+
+    async loadFeaturedContent() {
+        // Load hero featured
+        const featured = await this.api.getApps({ featured: true, limit: 4 });
+        if (!featured || !featured.length) return;
+
+        // Hero card (first featured)
+        const hero = featured[0];
+        const heroCard = document.getElementById('featured-hero');
+        if (hero) {
+            const imageUrl = hero.image || '';
+            heroCard.innerHTML = `
+                <div class="hero-image" ${imageUrl ? `style="background-image: url('${imageUrl}')"` : ''}>
+                    ${!imageUrl ? `[${hero.category || 'APP'}]` : ''}
+                </div>
+                <div class="hero-content">
+                    <span class="hero-badge">${hero.type || 'PAID'}</span>
+                    <h2 class="hero-title">${hero.name}</h2>
+                    <p class="hero-description">${hero.description}</p>
+                    <div class="hero-meta">
+                        <span>★ ${hero.rating || 0}/5</span>
+                        <span>${hero.downloads || 0} downloads</span>
+                    </div>
+                </div>
+            `;
+            heroCard.onclick = () => this.showAppDetail(hero);
+        }
+
+        // Secondary featured cards
+        const secondary = document.getElementById('featured-secondary');
+        secondary.innerHTML = '';
+        if (featured.length > 1) {
+            featured.slice(1, 4).forEach(app => {
+                const card = document.createElement('div');
+                card.className = 'secondary-card';
+                const imageUrl = app.image || '';
+                card.innerHTML = `
+                    <div class="secondary-image" ${imageUrl ? `style="background-image: url('${imageUrl}')"` : ''}>
+                        ${!imageUrl ? `[${app.category || 'APP'}]` : ''}
+                    </div>
+                    <div class="secondary-content">
+                        <h3 class="secondary-title">${app.name}</h3>
+                        <p class="secondary-desc">${(app.description || '').substring(0, 100)}...</p>
+                        <div class="secondary-meta">
+                            <span>${app.type || 'Open Source'}</span> · <span>★ ${app.rating || 0}/5</span>
+                        </div>
+                    </div>
+                `;
+                card.onclick = () => this.showAppDetail(app);
+                secondary.appendChild(card);
+            });
+        }
+    }
+
+    async loadSponsors() {
+        const sponsors = await this.api.getSponsors();
+        if (!sponsors || !sponsors.length) {
+            // Show placeholder if no sponsors
+            const container = document.getElementById('sponsored-content');
+            container.innerHTML = `
+                <div class="sponsor-card">
+                    <h4>Become a Sponsor</h4>
+                    <p>Reach thousands of developers using Crawl4AI</p>
+                    <a href="mailto:sponsors@crawl4ai.com">Contact Us →</a>
+                </div>
+            `;
+            return;
+        }
+
+        const container = document.getElementById('sponsored-content');
+        container.innerHTML = sponsors.slice(0, 5).map(sponsor => `
+            <div class="sponsor-card">
+                <h4>${sponsor.company_name}</h4>
+                <p>${sponsor.tier} Sponsor - Premium Solutions</p>
+                <a href="${sponsor.landing_url}" target="_blank">Learn More →</a>
+            </div>
+        `).join('');
+    }
+
+    async loadMainContent() {
+        // Load apps column
+        const apps = await this.api.getApps({ limit: 8 });
+        if (apps && apps.length) {
+            const appsGrid = document.getElementById('apps-grid');
+            appsGrid.innerHTML = apps.map(app => `
+                <div class="app-compact" onclick="marketplace.showAppDetail(${JSON.stringify(app).replace(/"/g, '&quot;')})">
+                    <div class="app-compact-header">
+                        <span>${app.category}</span>
+                        <span>★ ${app.rating}/5</span>
+                    </div>
+                    <div class="app-compact-title">${app.name}</div>
+                    <div class="app-compact-desc">${app.description}</div>
+                </div>
+            `).join('');
+        }
+
+        // Load articles column
+        const articles = await this.api.getArticles({ limit: 6 });
+        if (articles && articles.length) {
+            const articlesList = document.getElementById('articles-list');
+            articlesList.innerHTML = articles.map(article => `
+                <div class="article-compact" onclick="marketplace.showArticle('${article.id}')">
+                    <div class="article-meta">
+                        <span>${article.category}</span> · <span>${new Date(article.published_at).toLocaleDateString()}</span>
+                    </div>
+                    <div class="article-title">${article.title}</div>
+                    <div class="article-author">by ${article.author}</div>
+                </div>
+            `).join('');
+        }
+
+        // Load trending
+        if (apps && apps.length) {
+            const trending = apps.slice(0, 5);
+            const trendingList = document.getElementById('trending-list');
+            trendingList.innerHTML = trending.map((app, i) => `
+                <div class="trending-item" onclick="marketplace.showAppDetail(${JSON.stringify(app).replace(/"/g, '&quot;')})">
+                    <div class="trending-rank">${i + 1}</div>
+                    <div class="trending-info">
+                        <div class="trending-name">${app.name}</div>
+                        <div class="trending-stats">${app.downloads} downloads</div>
+                    </div>
+                </div>
+            `).join('');
+        }
+
+        // Load more apps grid
+        const moreApps = await this.api.getApps({ offset: 8, limit: 12 });
+        if (moreApps && moreApps.length) {
+            const moreGrid = document.getElementById('more-apps-grid');
+            moreGrid.innerHTML = moreApps.map(app => `
+                <div class="app-compact" onclick="marketplace.showAppDetail(${JSON.stringify(app).replace(/"/g, '&quot;')})">
+                    <div class="app-compact-header">
+                        <span>${app.category}</span>
+                        <span>${app.type}</span>
+                    </div>
+                    <div class="app-compact-title">${app.name}</div>
+                </div>
+            `).join('');
+        }
+    }
+
+    setupEventListeners() {
+        // Search
+        const searchInput = document.getElementById('search-input');
+        searchInput.addEventListener('input', (e) => {
+            clearTimeout(this.searchTimeout);
+            this.searchTimeout = setTimeout(() => this.search(e.target.value), 300);
+        });
+
+        // Keyboard shortcut
+        document.addEventListener('keydown', (e) => {
+            if (e.key === '/' && !searchInput.contains(document.activeElement)) {
+                e.preventDefault();
+                searchInput.focus();
+            }
+            if (e.key === 'Escape' && searchInput.contains(document.activeElement)) {
+                searchInput.blur();
+                searchInput.value = '';
+            }
+        });
+
+        // Type filter
+        const typeFilter = document.getElementById('type-filter');
+        typeFilter.addEventListener('change', (e) => {
+            this.currentType = e.target.value;
+            this.loadMainContent();
+        });
+
+        // Load more
+        const loadMore = document.getElementById('load-more');
+        loadMore.addEventListener('click', () => this.loadMoreApps());
+    }
+
+    async filterByCategory(category) {
+        // Update active state
+        document.querySelectorAll('.filter-btn').forEach(btn => {
+            btn.classList.toggle('active', btn.dataset.category === category);
+        });
+
+        this.currentCategory = category;
+        await this.loadMainContent();
+    }
+
+    async search(query) {
+        if (!query) {
+            await this.loadMainContent();
+            return;
+        }
+
+        const results = await this.api.search(query);
+        if (!results) return;
+
+        // Update apps grid with search results
+        if (results.apps && results.apps.length) {
+            const appsGrid = document.getElementById('apps-grid');
+            appsGrid.innerHTML = results.apps.map(app => `
+                <div class="app-compact" onclick="marketplace.showAppDetail(${JSON.stringify(app).replace(/"/g, '&quot;')})">
+                    <div class="app-compact-header">
+                        <span>${app.category}</span>
+                        <span>★ ${app.rating}/5</span>
+                    </div>
+                    <div class="app-compact-title">${app.name}</div>
+                    <div class="app-compact-desc">${app.description}</div>
+                </div>
+            `).join('');
+        }
+
+        // Update articles with search results
+        if (results.articles && results.articles.length) {
+            const articlesList = document.getElementById('articles-list');
+            articlesList.innerHTML = results.articles.map(article => `
+                <div class="article-compact" onclick="marketplace.showArticle('${article.id}')">
+                    <div class="article-meta">
+                        <span>${article.category}</span> · <span>${new Date(article.published_at).toLocaleDateString()}</span>
+                    </div>
+                    <div class="article-title">${article.title}</div>
+                    <div class="article-author">by ${article.author}</div>
+                </div>
+            `).join('');
+        }
+    }
+
+    async loadMoreApps() {
+        this.loadedApps += 12;
+        const moreApps = await this.api.getApps({ offset: this.loadedApps, limit: 12 });
+        if (moreApps && moreApps.length) {
+            const moreGrid = document.getElementById('more-apps-grid');
+            moreApps.forEach(app => {
+                const card = document.createElement('div');
+                card.className = 'app-compact';
+                card.innerHTML = `
+                    <div class="app-compact-header">
+                        <span>${app.category}</span>
+                        <span>${app.type}</span>
+                    </div>
+                    <div class="app-compact-title">${app.name}</div>
+                `;
+                card.onclick = () => this.showAppDetail(app);
+                moreGrid.appendChild(card);
+            });
+        }
+    }
+
+    showAppDetail(app) {
+        // Navigate to detail page instead of showing modal
+        const slug = app.slug || app.name.toLowerCase().replace(/\s+/g, '-');
+        window.location.href = `app-detail.html?app=${slug}`;
+    }
+
+    showArticle(articleId) {
+        // Could create article detail page similarly
+        console.log('Show article:', articleId);
+    }
+}
+
+// Initialize marketplace
+let marketplace;
+document.addEventListener('DOMContentLoaded', () => {
+    marketplace = new MarketplaceUI();
+});

docs/md_v2/marketplace/index.html

@@ -0,0 +1,147 @@
+<!DOCTYPE html>
+<html lang="en" data-theme="dark">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Marketplace - Crawl4AI</title>
+    <link rel="stylesheet" href="marketplace.css">
+</head>
+<body>
+    <div class="marketplace-container">
+        <!-- Header -->
+        <header class="marketplace-header">
+            <div class="header-content">
+                <div class="header-left">
+                    <div class="logo-title">
+                        <img src="../assets/images/logo.png" alt="Crawl4AI" class="header-logo">
+                        <h1>
+                            <span class="ascii-border">[</span>
+                            Marketplace
+                            <span class="ascii-border">]</span>
+                        </h1>
+                    </div>
+                    <p class="tagline">Tools, Integrations & Resources for Web Crawling</p>
+                </div>
+                <div class="header-stats" id="stats">
+                    <span class="stat-item">Apps: <span id="total-apps">--</span></span>
+                    <span class="stat-item">Articles: <span id="total-articles">--</span></span>
+                    <span class="stat-item">Downloads: <span id="total-downloads">--</span></span>
+                </div>
+            </div>
+        </header>
+
+        <!-- Search and Category Bar -->
+        <div class="search-filter-bar">
+            <div class="search-box">
+                <span class="search-icon">></span>
+                <input type="text" id="search-input" placeholder="Search apps, articles, tools..." />
+                <kbd>/</kbd>
+            </div>
+            <div class="category-filter" id="category-filter">
+                <button class="filter-btn active" data-category="all">All</button>
+                <!-- Categories will be loaded here -->
+            </div>
+        </div>
+
+        <!-- Magazine Grid Layout -->
+        <main class="magazine-layout">
+            <!-- Hero Featured Section -->
+            <section class="hero-featured">
+                <div id="featured-hero" class="featured-hero-card">
+                    <!-- Large featured card with big image -->
+                </div>
+            </section>
+
+            <!-- Secondary Featured -->
+            <section class="secondary-featured">
+                <div id="featured-secondary" class="featured-secondary-cards">
+                    <!-- 2-3 medium featured cards with images -->
+                </div>
+            </section>
+
+            <!-- Sponsored Section -->
+            <section class="sponsored-section">
+                <div class="section-label">SPONSORED</div>
+                <div id="sponsored-content" class="sponsored-cards">
+                    <!-- Sponsored content cards -->
+                </div>
+            </section>
+
+            <!-- Main Content Grid -->
+            <section class="main-content">
+                <!-- Apps Column -->
+                <div class="apps-column">
+                    <div class="column-header">
+                        <h2><span class="ascii-icon">></span> Latest Apps</h2>
+                        <select id="type-filter" class="mini-filter">
+                            <option value="">All</option>
+                            <option value="Open Source">Open Source</option>
+                            <option value="Paid">Paid</option>
+                        </select>
+                    </div>
+                    <div id="apps-grid" class="apps-compact-grid">
+                        <!-- Compact app cards -->
+                    </div>
+                </div>
+
+                <!-- Articles Column -->
+                <div class="articles-column">
+                    <div class="column-header">
+                        <h2><span class="ascii-icon">></span> Latest Articles</h2>
+                    </div>
+                    <div id="articles-list" class="articles-compact-list">
+                        <!-- Article items -->
+                    </div>
+                </div>
+
+                <!-- Trending/Tools Column -->
+                <div class="trending-column">
+                    <div class="column-header">
+                        <h2><span class="ascii-icon">#</span> Trending</h2>
+                    </div>
+                    <div id="trending-list" class="trending-items">
+                        <!-- Trending items -->
+                    </div>
+
+                    <div class="submit-box">
+                        <h3><span class="ascii-icon">+</span> Submit Your Tool</h3>
+                        <p>Share your integration</p>
+                        <a href="mailto:marketplace@crawl4ai.com" class="submit-btn">Submit →</a>
+                    </div>
+                </div>
+            </section>
+
+            <!-- More Apps Grid -->
+            <section class="more-apps">
+                <div class="section-header">
+                    <h2><span class="ascii-icon">></span> More Apps</h2>
+                    <button id="load-more" class="load-more-btn">Load More ↓</button>
+                </div>
+                <div id="more-apps-grid" class="more-apps-grid">
+                    <!-- Additional app cards -->
+                </div>
+            </section>
+        </main>
+
+        <!-- Footer -->
+        <footer class="marketplace-footer">
+            <div class="footer-content">
+                <div class="footer-section">
+                    <h3>About Marketplace</h3>
+                    <p>Discover tools and integrations built by the Crawl4AI community.</p>
+                </div>
+                <div class="footer-section">
+                    <h3>Become a Sponsor</h3>
+                    <p>Reach developers building with Crawl4AI</p>
+                    <a href="mailto:sponsors@crawl4ai.com" class="sponsor-btn">Learn More →</a>
+                </div>
+            </div>
+            <div class="footer-bottom">
+                <p>[ Crawl4AI Marketplace · Updated <span id="last-update">--</span> ]</p>
+            </div>
+        </footer>
+    </div>
+
+    <script src="marketplace.js"></script>
+</body>
+</html>

docs/md_v2/marketplace/marketplace.css

@@ -0,0 +1,994 @@
+/* Marketplace CSS - Magazine Style Terminal Theme */
+@import url('../../assets/styles.css');
+
+:root {
+    --primary-cyan: #50ffff;
+    --primary-teal: #09b5a5;
+    --accent-pink: #f380f5;
+    --bg-dark: #070708;
+    --bg-secondary: #1a1a1a;
+    --bg-tertiary: #3f3f44;
+    --text-primary: #e8e9ed;
+    --text-secondary: #d5cec0;
+    --text-tertiary: #a3abba;
+    --border-color: #3f3f44;
+    --success: #50ff50;
+    --error: #ff3c74;
+    --warning: #f59e0b;
+}
+
+* {
+    margin: 0;
+    padding: 0;
+    box-sizing: border-box;
+}
+
+body {
+    font-family: 'Dank Mono', Monaco, monospace;
+    background: var(--bg-dark);
+    color: var(--text-primary);
+    line-height: 1.6;
+}
+
+/* Global link styles */
+a {
+    color: var(--primary-cyan);
+    text-decoration: none;
+    transition: color 0.2s;
+}
+
+a:hover {
+    color: var(--accent-pink);
+}
+
+.marketplace-container {
+    min-height: 100vh;
+}
+
+/* Header */
+.marketplace-header {
+    background: var(--bg-secondary);
+    border-bottom: 1px solid var(--border-color);
+    padding: 1.5rem 0;
+}
+
+.header-content {
+    max-width: 1800px;
+    margin: 0 auto;
+    padding: 0 2rem;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+}
+
+.logo-title {
+    display: flex;
+    align-items: center;
+    gap: 1rem;
+}
+
+.header-logo {
+    height: 40px;
+    width: auto;
+    filter: brightness(1.2);
+}
+
+.marketplace-header h1 {
+    font-size: 1.5rem;
+    color: var(--primary-cyan);
+    margin: 0;
+}
+
+.ascii-border {
+    color: var(--border-color);
+}
+
+.tagline {
+    font-size: 0.875rem;
+    color: var(--text-tertiary);
+    margin-top: 0.25rem;
+}
+
+.header-stats {
+    display: flex;
+    gap: 2rem;
+}
+
+.stat-item {
+    font-size: 0.875rem;
+    color: var(--text-secondary);
+}
+
+.stat-item span {
+    color: var(--primary-cyan);
+    font-weight: 600;
+}
+
+/* Search and Filter Bar */
+.search-filter-bar {
+    max-width: 1800px;
+    margin: 1.5rem auto;
+    padding: 0 2rem;
+    display: flex;
+    gap: 1rem;
+    align-items: center;
+}
+
+.search-box {
+    flex: 1;
+    max-width: 500px;
+    display: flex;
+    align-items: center;
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    padding: 0.75rem 1rem;
+    transition: border-color 0.2s;
+}
+
+.search-box:focus-within {
+    border-color: var(--primary-cyan);
+}
+
+.search-icon {
+    color: var(--text-tertiary);
+    margin-right: 1rem;
+}
+
+#search-input {
+    flex: 1;
+    background: transparent;
+    border: none;
+    color: var(--text-primary);
+    font-family: inherit;
+    font-size: 0.9rem;
+    outline: none;
+}
+
+.search-box kbd {
+    font-size: 0.75rem;
+    padding: 0.2rem 0.5rem;
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    color: var(--text-tertiary);
+}
+
+.category-filter {
+    display: flex;
+    gap: 0.5rem;
+    flex-wrap: wrap;
+}
+
+.filter-btn {
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    padding: 0.5rem 1rem;
+    font-family: inherit;
+    font-size: 0.875rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.filter-btn:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+.filter-btn.active {
+    background: var(--primary-cyan);
+    color: var(--bg-dark);
+    border-color: var(--primary-cyan);
+}
+
+/* Magazine Layout */
+.magazine-layout {
+    max-width: 1800px;
+    margin: 0 auto;
+    padding: 0 2rem 4rem;
+    display: grid;
+    grid-template-columns: 1fr;
+    gap: 2rem;
+}
+
+/* Hero Featured Section */
+.hero-featured {
+    grid-column: 1 / -1;
+    position: relative;
+}
+
+.hero-featured::before {
+    content: '';
+    position: absolute;
+    top: -20px;
+    left: -20px;
+    right: -20px;
+    bottom: -20px;
+    background: radial-gradient(ellipse at center, rgba(80, 255, 255, 0.05), transparent 70%);
+    pointer-events: none;
+    z-index: -1;
+}
+
+.featured-hero-card {
+    background: linear-gradient(135deg, #1a1a2e, #0f0f1e);
+    border: 2px solid var(--primary-cyan);
+    box-shadow: 0 0 30px rgba(80, 255, 255, 0.15),
+                inset 0 0 20px rgba(80, 255, 255, 0.05);
+    height: 380px;
+    position: relative;
+    overflow: hidden;
+    cursor: pointer;
+    transition: all 0.3s ease;
+    display: flex;
+    flex-direction: column;
+}
+
+.featured-hero-card:hover {
+    border-color: var(--accent-pink);
+    box-shadow: 0 0 40px rgba(243, 128, 245, 0.2),
+                inset 0 0 30px rgba(243, 128, 245, 0.05);
+    transform: translateY(-2px);
+}
+
+.hero-image {
+    width: 100%;
+    height: 200px;
+    min-height: 200px;
+    max-height: 200px;
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.1), rgba(243, 128, 245, 0.05));
+    background-size: cover;
+    background-position: center;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-size: 3rem;
+    color: var(--primary-cyan);
+    flex-shrink: 0;
+    position: relative;
+    filter: brightness(1.1) contrast(1.1);
+    overflow: hidden;
+}
+
+.hero-image img {
+    width: 100%;
+    height: 100%;
+    object-fit: cover;
+    object-position: center;
+}
+
+.hero-image::after {
+    content: '';
+    position: absolute;
+    bottom: 0;
+    left: 0;
+    right: 0;
+    height: 60%;
+    background: linear-gradient(to top, rgba(10, 10, 20, 0.95), transparent);
+}
+
+.hero-content {
+    padding: 1.5rem;
+    flex: 1;
+    display: flex;
+    flex-direction: column;
+    justify-content: space-between;
+}
+
+.hero-badge {
+    display: inline-block;
+    padding: 0.3rem 0.6rem;
+    background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
+    color: var(--bg-dark);
+    font-size: 0.7rem;
+    text-transform: uppercase;
+    margin-bottom: 0.5rem;
+    font-weight: 600;
+    box-shadow: 0 2px 10px rgba(80, 255, 255, 0.3);
+}
+
+.hero-title {
+    font-size: 1.6rem;
+    color: var(--primary-cyan);
+    margin: 0.5rem 0;
+    text-shadow: 0 0 20px rgba(80, 255, 255, 0.5);
+}
+
+.hero-description {
+    color: var(--text-secondary);
+    line-height: 1.5;
+}
+
+.hero-meta {
+    display: flex;
+    gap: 1.5rem;
+    margin-top: 1rem;
+    font-size: 0.875rem;
+}
+
+.hero-meta span {
+    color: var(--text-tertiary);
+}
+
+.hero-meta span:first-child {
+    color: var(--warning);
+}
+
+/* Secondary Featured */
+.secondary-featured {
+    grid-column: 1 / -1;
+    min-height: 380px;
+    display: flex;
+    align-items: flex-start;
+}
+
+.featured-secondary-cards {
+    width: 100%;
+    display: flex;
+    flex-direction: column;
+    gap: 0.75rem;
+    align-items: stretch;
+}
+
+.secondary-card {
+    background: linear-gradient(135deg, rgba(80, 255, 255, 0.03), rgba(243, 128, 245, 0.02));
+    border: 1px solid rgba(80, 255, 255, 0.3);
+    cursor: pointer;
+    transition: all 0.3s ease;
+    display: flex;
+    overflow: hidden;
+    height: 118px;
+    min-height: 118px;
+    max-height: 118px;
+    flex-shrink: 0;
+    box-shadow: 0 2px 10px rgba(0, 0, 0, 0.3);
+}
+
+.secondary-card:hover {
+    border-color: var(--accent-pink);
+    background: linear-gradient(135deg, rgba(243, 128, 245, 0.05), rgba(80, 255, 255, 0.03));
+    box-shadow: 0 4px 15px rgba(243, 128, 245, 0.2);
+    transform: translateX(-3px);
+}
+
+.secondary-image {
+    width: 120px;
+    background: linear-gradient(135deg, var(--bg-tertiary), var(--bg-secondary));
+    background-size: cover;
+    background-position: center;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-size: 1.5rem;
+    color: var(--primary-cyan);
+    flex-shrink: 0;
+}
+
+.secondary-content {
+    flex: 1;
+    padding: 1rem;
+    display: flex;
+    flex-direction: column;
+    justify-content: space-between;
+}
+
+.secondary-title {
+    font-size: 1rem;
+    color: var(--text-primary);
+    margin-bottom: 0.25rem;
+}
+
+.secondary-desc {
+    font-size: 0.75rem;
+    color: var(--text-secondary);
+    display: -webkit-box;
+    -webkit-line-clamp: 2;
+    -webkit-box-orient: vertical;
+    overflow: hidden;
+}
+
+.secondary-meta {
+    font-size: 0.75rem;
+    color: var(--text-tertiary);
+}
+
+.secondary-meta span:last-child {
+    color: var(--warning);
+}
+
+/* Sponsored Section */
+.sponsored-section {
+    grid-column: 1 / -1;
+    background: var(--bg-secondary);
+    border: 1px solid var(--warning);
+    padding: 1rem;
+    position: relative;
+}
+
+.section-label {
+    position: absolute;
+    top: -0.5rem;
+    left: 1rem;
+    background: var(--bg-secondary);
+    padding: 0 0.5rem;
+    color: var(--warning);
+    font-size: 0.65rem;
+    letter-spacing: 0.1em;
+}
+
+.sponsored-cards {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+    gap: 1rem;
+}
+
+.sponsor-card {
+    padding: 1rem;
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+}
+
+.sponsor-logo {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    height: 60px;
+    margin-bottom: 0.75rem;
+}
+
+.sponsor-logo img {
+    max-height: 60px;
+    max-width: 100%;
+    width: auto;
+    object-fit: contain;
+}
+
+.sponsor-card h4 {
+    color: var(--accent-pink);
+    margin-bottom: 0.5rem;
+}
+
+.sponsor-card p {
+    color: var(--text-secondary);
+    font-size: 0.85rem;
+    margin-bottom: 0.75rem;
+}
+
+.sponsor-card a {
+    color: var(--primary-cyan);
+    text-decoration: none;
+    font-size: 0.85rem;
+}
+
+.sponsor-card a:hover {
+    color: var(--accent-pink);
+}
+
+/* Main Content Grid */
+.main-content {
+    grid-column: 1 / -1;
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
+    gap: 2rem;
+}
+
+/* Column Headers */
+.column-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 1rem;
+    border-bottom: 1px solid var(--border-color);
+    padding-bottom: 0.5rem;
+}
+
+.column-header h2 {
+    font-size: 1.1rem;
+    color: var(--text-primary);
+}
+
+.mini-filter {
+    background: var(--bg-tertiary);
+    border: 1px solid var(--border-color);
+    color: var(--text-primary);
+    padding: 0.25rem 0.5rem;
+    font-family: inherit;
+    font-size: 0.75rem;
+}
+
+.ascii-icon {
+    color: var(--primary-cyan);
+}
+
+/* Apps Column */
+.apps-compact-grid {
+    display: flex;
+    flex-direction: column;
+    gap: 0.75rem;
+}
+
+.app-compact {
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    border-left: 3px solid var(--border-color);
+    padding: 0.75rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.app-compact:hover {
+    border-color: var(--primary-cyan);
+    border-left-color: var(--accent-pink);
+    transform: translateX(2px);
+}
+
+.app-compact-header {
+    display: flex;
+    justify-content: space-between;
+    font-size: 0.75rem;
+    color: var(--text-tertiary);
+    margin-bottom: 0.25rem;
+}
+
+.app-compact-header span:first-child {
+    color: var(--primary-cyan);
+}
+
+.app-compact-header span:last-child {
+    color: var(--warning);
+}
+
+.app-compact-title {
+    font-size: 0.9rem;
+    color: var(--text-primary);
+    margin-bottom: 0.25rem;
+}
+
+.app-compact-desc {
+    font-size: 0.75rem;
+    color: var(--text-secondary);
+    display: -webkit-box;
+    -webkit-line-clamp: 2;
+    -webkit-box-orient: vertical;
+    overflow: hidden;
+}
+
+/* Articles Column */
+.articles-compact-list {
+    display: flex;
+    flex-direction: column;
+    gap: 1rem;
+}
+
+.article-compact {
+    border-left: 2px solid var(--border-color);
+    padding-left: 1rem;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.article-compact:hover {
+    border-left-color: var(--primary-cyan);
+}
+
+.article-meta {
+    font-size: 0.7rem;
+    color: var(--text-tertiary);
+    margin-bottom: 0.25rem;
+}
+
+.article-meta span:first-child {
+    color: var(--accent-pink);
+}
+
+.article-title {
+    font-size: 0.9rem;
+    color: var(--text-primary);
+    margin-bottom: 0.25rem;
+}
+
+.article-author {
+    font-size: 0.75rem;
+    color: var(--text-secondary);
+}
+
+/* Trending Column */
+.trending-items {
+    display: flex;
+    flex-direction: column;
+    gap: 0.5rem;
+}
+
+.trending-item {
+    display: flex;
+    align-items: center;
+    gap: 0.75rem;
+    padding: 0.5rem;
+    background: var(--bg-secondary);
+    border: 1px solid var(--border-color);
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.trending-item:hover {
+    border-color: var(--primary-cyan);
+}
+
+.trending-rank {
+    font-size: 1.2rem;
+    color: var(--primary-cyan);
+    width: 2rem;
+    text-align: center;
+}
+
+.trending-info {
+    flex: 1;
+}
+
+.trending-name {
+    font-size: 0.85rem;
+    color: var(--text-primary);
+}
+
+.trending-stats {
+    font-size: 0.7rem;
+    color: var(--text-tertiary);
+}
+
+/* Submit Box */
+.submit-box {
+    margin-top: 1.5rem;
+    background: var(--bg-secondary);
+    border: 1px solid var(--primary-cyan);
+    padding: 1rem;
+    text-align: center;
+}
+
+.submit-box h3 {
+    font-size: 1rem;
+    color: var(--primary-cyan);
+    margin-bottom: 0.5rem;
+}
+
+.submit-box p {
+    font-size: 0.8rem;
+    color: var(--text-secondary);
+    margin-bottom: 0.75rem;
+}
+
+.submit-btn {
+    display: inline-block;
+    padding: 0.5rem 1rem;
+    background: transparent;
+    border: 1px solid var(--primary-cyan);
+    color: var(--primary-cyan);
+    text-decoration: none;
+    transition: all 0.2s;
+}
+
+.submit-btn:hover {
+    background: var(--primary-cyan);
+    color: var(--bg-dark);
+}
+
+/* More Apps Section */
+.more-apps {
+    grid-column: 1 / -1;
+    margin-top: 2rem;
+}
+
+.section-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 1rem;
+}
+
+.more-apps-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
+    gap: 1rem;
+}
+
+.load-more-btn {
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-secondary);
+    padding: 0.5rem 1.5rem;
+    font-family: inherit;
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.load-more-btn:hover {
+    border-color: var(--primary-cyan);
+    color: var(--primary-cyan);
+}
+
+/* Footer */
+.marketplace-footer {
+    background: var(--bg-secondary);
+    border-top: 1px solid var(--border-color);
+    margin-top: 4rem;
+    padding: 2rem 0;
+}
+
+.footer-content {
+    max-width: 1800px;
+    margin: 0 auto;
+    padding: 0 2rem;
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    gap: 2rem;
+}
+
+.footer-section h3 {
+    font-size: 1rem;
+    margin-bottom: 0.5rem;
+    color: var(--primary-cyan);
+}
+
+.footer-section p {
+    font-size: 0.875rem;
+    color: var(--text-secondary);
+    margin-bottom: 1rem;
+}
+
+.sponsor-btn {
+    display: inline-block;
+    padding: 0.5rem 1rem;
+    background: transparent;
+    border: 1px solid var(--primary-cyan);
+    color: var(--primary-cyan);
+    text-decoration: none;
+    transition: all 0.2s;
+}
+
+.sponsor-btn:hover {
+    background: var(--primary-cyan);
+    color: var(--bg-dark);
+}
+
+.footer-bottom {
+    max-width: 1800px;
+    margin: 2rem auto 0;
+    padding: 1rem 2rem 0;
+    border-top: 1px solid var(--border-color);
+    font-size: 0.75rem;
+    color: var(--text-tertiary);
+}
+
+/* Modal */
+.modal {
+    position: fixed;
+    top: 0;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    background: rgba(0, 0, 0, 0.8);
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    z-index: 1000;
+}
+
+.modal.hidden {
+    display: none;
+}
+
+.modal-content {
+    background: var(--bg-secondary);
+    border: 1px solid var(--primary-cyan);
+    max-width: 800px;
+    width: 90%;
+    max-height: 80vh;
+    overflow-y: auto;
+    position: relative;
+}
+
+.modal-close {
+    position: absolute;
+    top: 1rem;
+    right: 1rem;
+    background: transparent;
+    border: 1px solid var(--border-color);
+    color: var(--text-primary);
+    padding: 0.25rem 0.5rem;
+    cursor: pointer;
+    font-size: 1.2rem;
+}
+
+.modal-close:hover {
+    border-color: var(--error);
+    color: var(--error);
+}
+
+.app-detail {
+    padding: 2rem;
+}
+
+.app-detail h2 {
+    font-size: 1.5rem;
+    margin-bottom: 1rem;
+    color: var(--primary-cyan);
+}
+
+/* Loading */
+.loading {
+    text-align: center;
+    padding: 2rem;
+    color: var(--text-tertiary);
+}
+
+.no-results {
+    text-align: center;
+    padding: 2rem;
+    color: var(--text-tertiary);
+}
+
+/* Responsive - Tablet */
+@media (min-width: 768px) {
+    .magazine-layout {
+        grid-template-columns: repeat(2, 1fr);
+    }
+
+    .hero-featured {
+        grid-column: 1 / -1;
+    }
+
+    .secondary-featured {
+        grid-column: 1 / -1;
+    }
+
+    .sponsored-section {
+        grid-column: 1 / -1;
+    }
+
+    .main-content {
+        grid-column: 1 / -1;
+        grid-template-columns: repeat(2, 1fr);
+    }
+}
+
+/* Responsive - Desktop */
+@media (min-width: 1024px) {
+    .magazine-layout {
+        grid-template-columns: repeat(3, 1fr);
+    }
+
+    .hero-featured {
+        grid-column: 1 / 3;
+        grid-row: 1;
+    }
+
+    .secondary-featured {
+        grid-column: 3 / 4;
+        grid-row: 1;
+    }
+
+    .featured-secondary-cards {
+        flex-direction: column;
+    }
+
+    .sponsored-section {
+        grid-column: 1 / -1;
+    }
+
+    .main-content {
+        grid-column: 1 / -1;
+        grid-template-columns: repeat(3, 1fr);
+    }
+}
+
+/* Responsive - Wide Desktop */
+@media (min-width: 1400px) {
+    .magazine-layout {
+        grid-template-columns: repeat(4, 1fr);
+    }
+
+    .hero-featured {
+        grid-column: 1 / 3;
+    }
+
+    .secondary-featured {
+        grid-column: 3 / 5;
+        grid-row: 1;
+        min-height: auto;
+    }
+
+    .featured-secondary-cards {
+        display: grid;
+        grid-template-columns: repeat(2, 1fr);
+        flex-direction: unset;
+    }
+
+    .main-content {
+        grid-template-columns: repeat(4, 1fr);
+    }
+
+    .apps-column {
+        grid-column: span 2;
+    }
+
+    .more-apps-grid {
+        grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
+    }
+}
+
+/* Responsive - Ultra Wide Desktop (for coders with wide monitors) */
+@media (min-width: 1800px) {
+    .magazine-layout {
+        grid-template-columns: repeat(5, 1fr);
+    }
+
+    .hero-featured {
+        grid-column: 1 / 3;
+    }
+
+    .secondary-featured {
+        grid-column: 3 / 6;
+        min-height: auto;
+    }
+
+    .featured-secondary-cards {
+        display: grid;
+        grid-template-columns: repeat(3, 1fr);
+        flex-direction: unset;
+    }
+
+    .sponsored-section {
+        grid-column: 1 / -1;
+    }
+
+    .sponsored-cards {
+        grid-template-columns: repeat(5, 1fr);
+    }
+
+    .main-content {
+        grid-template-columns: repeat(5, 1fr);
+    }
+
+    .apps-column {
+        grid-column: span 2;
+    }
+
+    .articles-column {
+        grid-column: span 2;
+    }
+
+    .more-apps-grid {
+        grid-template-columns: repeat(auto-fill, minmax(300px, 1fr));
+    }
+}
+
+/* Responsive - Mobile */
+@media (max-width: 767px) {
+    .header-content {
+        flex-direction: column;
+        gap: 1rem;
+    }
+
+    .search-filter-bar {
+        flex-direction: column;
+        align-items: stretch;
+    }
+
+    .search-box {
+        max-width: none;
+    }
+
+    .magazine-layout {
+        padding: 0 1rem 2rem;
+    }
+
+    .footer-content {
+        grid-template-columns: 1fr;
+    }
+
+    .secondary-card {
+        flex-direction: column;
+    }
+
+    .secondary-image {
+        width: 100%;
+        height: 150px;
+    }
+}

docs/md_v2/marketplace/marketplace.js

@@ -0,0 +1,412 @@
+// Marketplace JS - Magazine Layout
+const { API_BASE, API_ORIGIN } = (() => {
+    const { hostname, port } = window.location;
+    if ((hostname === 'localhost' || hostname === '127.0.0.1') && port === '8000') {
+        const origin = 'http://127.0.0.1:8100';
+        return { API_BASE: `${origin}/marketplace/api`, API_ORIGIN: origin };
+    }
+    return { API_BASE: '/marketplace/api', API_ORIGIN: '' };
+})();
+
+const resolveAssetUrl = (path) => {
+    if (!path) return '';
+    if (/^https?:\/\//i.test(path)) return path;
+    if (path.startsWith('/') && API_ORIGIN) {
+        return `${API_ORIGIN}${path}`;
+    }
+    return path;
+};
+const CACHE_TTL = 3600000; // 1 hour in ms
+
+class MarketplaceCache {
+    constructor() {
+        this.prefix = 'c4ai_market_';
+    }
+
+    get(key) {
+        const item = localStorage.getItem(this.prefix + key);
+        if (!item) return null;
+
+        const data = JSON.parse(item);
+        if (Date.now() > data.expires) {
+            localStorage.removeItem(this.prefix + key);
+            return null;
+        }
+        return data.value;
+    }
+
+    set(key, value, ttl = CACHE_TTL) {
+        const data = {
+            value: value,
+            expires: Date.now() + ttl
+        };
+        localStorage.setItem(this.prefix + key, JSON.stringify(data));
+    }
+
+    clear() {
+        Object.keys(localStorage)
+            .filter(k => k.startsWith(this.prefix))
+            .forEach(k => localStorage.removeItem(k));
+    }
+}
+
+class MarketplaceAPI {
+    constructor() {
+        this.cache = new MarketplaceCache();
+        this.searchTimeout = null;
+    }
+
+    async fetch(endpoint, useCache = true) {
+        const cacheKey = endpoint.replace(/[^\w]/g, '_');
+
+        if (useCache) {
+            const cached = this.cache.get(cacheKey);
+            if (cached) return cached;
+        }
+
+        try {
+            const response = await fetch(`${API_BASE}${endpoint}`);
+            if (!response.ok) throw new Error(`HTTP ${response.status}`);
+
+            const data = await response.json();
+            this.cache.set(cacheKey, data);
+            return data;
+        } catch (error) {
+            console.error('API Error:', error);
+            return null;
+        }
+    }
+
+    async getStats() {
+        return this.fetch('/stats');
+    }
+
+    async getCategories() {
+        return this.fetch('/categories');
+    }
+
+    async getApps(params = {}) {
+        const query = new URLSearchParams(params).toString();
+        return this.fetch(`/apps${query ? '?' + query : ''}`);
+    }
+
+    async getArticles(params = {}) {
+        const query = new URLSearchParams(params).toString();
+        return this.fetch(`/articles${query ? '?' + query : ''}`);
+    }
+
+    async getSponsors() {
+        return this.fetch('/sponsors');
+    }
+
+    async search(query) {
+        if (query.length < 2) return {};
+        return this.fetch(`/search?q=${encodeURIComponent(query)}`, false);
+    }
+}
+
+class MarketplaceUI {
+    constructor() {
+        this.api = new MarketplaceAPI();
+        this.currentCategory = 'all';
+        this.currentType = '';
+        this.searchTimeout = null;
+        this.loadedApps = 10;
+        this.init();
+    }
+
+    async init() {
+        await this.loadStats();
+        await this.loadCategories();
+        await this.loadFeaturedContent();
+        await this.loadSponsors();
+        await this.loadMainContent();
+        this.setupEventListeners();
+    }
+
+    async loadStats() {
+        const stats = await this.api.getStats();
+        if (stats) {
+            document.getElementById('total-apps').textContent = stats.total_apps || '0';
+            document.getElementById('total-articles').textContent = stats.total_articles || '0';
+            document.getElementById('total-downloads').textContent = stats.total_downloads || '0';
+            document.getElementById('last-update').textContent = new Date().toLocaleDateString();
+        }
+    }
+
+    async loadCategories() {
+        const categories = await this.api.getCategories();
+        if (!categories) return;
+
+        const filter = document.getElementById('category-filter');
+        categories.forEach(cat => {
+            const btn = document.createElement('button');
+            btn.className = 'filter-btn';
+            btn.dataset.category = cat.slug;
+            btn.textContent = cat.name;
+            btn.onclick = () => this.filterByCategory(cat.slug);
+            filter.appendChild(btn);
+        });
+    }
+
+    async loadFeaturedContent() {
+        // Load hero featured
+        const featured = await this.api.getApps({ featured: true, limit: 4 });
+        if (!featured || !featured.length) return;
+
+        // Hero card (first featured)
+        const hero = featured[0];
+        const heroCard = document.getElementById('featured-hero');
+        if (hero) {
+            const imageUrl = hero.image || '';
+            heroCard.innerHTML = `
+                <div class="hero-image" ${imageUrl ? `style="background-image: url('${imageUrl}')"` : ''}>
+                    ${!imageUrl ? `[${hero.category || 'APP'}]` : ''}
+                </div>
+                <div class="hero-content">
+                    <span class="hero-badge">${hero.type || 'PAID'}</span>
+                    <h2 class="hero-title">${hero.name}</h2>
+                    <p class="hero-description">${hero.description}</p>
+                    <div class="hero-meta">
+                        <span>★ ${hero.rating || 0}/5</span>
+                        <span>${hero.downloads || 0} downloads</span>
+                    </div>
+                </div>
+            `;
+            heroCard.onclick = () => this.showAppDetail(hero);
+        }
+
+        // Secondary featured cards
+        const secondary = document.getElementById('featured-secondary');
+        secondary.innerHTML = '';
+        if (featured.length > 1) {
+            featured.slice(1, 4).forEach(app => {
+                const card = document.createElement('div');
+                card.className = 'secondary-card';
+                const imageUrl = app.image || '';
+                card.innerHTML = `
+                    <div class="secondary-image" ${imageUrl ? `style="background-image: url('${imageUrl}')"` : ''}>
+                        ${!imageUrl ? `[${app.category || 'APP'}]` : ''}
+                    </div>
+                    <div class="secondary-content">
+                        <h3 class="secondary-title">${app.name}</h3>
+                        <p class="secondary-desc">${(app.description || '').substring(0, 100)}...</p>
+                        <div class="secondary-meta">
+                            <span>${app.type || 'Open Source'}</span> · <span>★ ${app.rating || 0}/5</span>
+                        </div>
+                    </div>
+                `;
+                card.onclick = () => this.showAppDetail(app);
+                secondary.appendChild(card);
+            });
+        }
+    }
+
+    async loadSponsors() {
+        const sponsors = await this.api.getSponsors();
+        if (!sponsors || !sponsors.length) {
+            // Show placeholder if no sponsors
+            const container = document.getElementById('sponsored-content');
+            container.innerHTML = `
+                <div class="sponsor-card">
+                    <h4>Become a Sponsor</h4>
+                    <p>Reach thousands of developers using Crawl4AI</p>
+                    <a href="mailto:sponsors@crawl4ai.com">Contact Us →</a>
+                </div>
+            `;
+            return;
+        }
+
+        const container = document.getElementById('sponsored-content');
+        container.innerHTML = sponsors.slice(0, 5).map(sponsor => `
+            <div class="sponsor-card">
+                ${sponsor.logo_url ? `<div class="sponsor-logo"><img src="${resolveAssetUrl(sponsor.logo_url)}" alt="${sponsor.company_name} logo"></div>` : ''}
+                <h4>${sponsor.company_name}</h4>
+                <p>${sponsor.tier} Sponsor - Premium Solutions</p>
+                <a href="${sponsor.landing_url}" target="_blank">Learn More →</a>
+            </div>
+        `).join('');
+    }
+
+    async loadMainContent() {
+        // Load apps column
+        const apps = await this.api.getApps({ limit: 8 });
+        if (apps && apps.length) {
+            const appsGrid = document.getElementById('apps-grid');
+            appsGrid.innerHTML = apps.map(app => `
+                <div class="app-compact" onclick="marketplace.showAppDetail(${JSON.stringify(app).replace(/"/g, '&quot;')})">
+                    <div class="app-compact-header">
+                        <span>${app.category}</span>
+                        <span>★ ${app.rating}/5</span>
+                    </div>
+                    <div class="app-compact-title">${app.name}</div>
+                    <div class="app-compact-desc">${app.description}</div>
+                </div>
+            `).join('');
+        }
+
+        // Load articles column
+        const articles = await this.api.getArticles({ limit: 6 });
+        if (articles && articles.length) {
+            const articlesList = document.getElementById('articles-list');
+            articlesList.innerHTML = articles.map(article => `
+                <div class="article-compact" onclick="marketplace.showArticle('${article.id}')">
+                    <div class="article-meta">
+                        <span>${article.category}</span> · <span>${new Date(article.published_at).toLocaleDateString()}</span>
+                    </div>
+                    <div class="article-title">${article.title}</div>
+                    <div class="article-author">by ${article.author}</div>
+                </div>
+            `).join('');
+        }
+
+        // Load trending
+        if (apps && apps.length) {
+            const trending = apps.slice(0, 5);
+            const trendingList = document.getElementById('trending-list');
+            trendingList.innerHTML = trending.map((app, i) => `
+                <div class="trending-item" onclick="marketplace.showAppDetail(${JSON.stringify(app).replace(/"/g, '&quot;')})">
+                    <div class="trending-rank">${i + 1}</div>
+                    <div class="trending-info">
+                        <div class="trending-name">${app.name}</div>
+                        <div class="trending-stats">${app.downloads} downloads</div>
+                    </div>
+                </div>
+            `).join('');
+        }
+
+        // Load more apps grid
+        const moreApps = await this.api.getApps({ offset: 8, limit: 12 });
+        if (moreApps && moreApps.length) {
+            const moreGrid = document.getElementById('more-apps-grid');
+            moreGrid.innerHTML = moreApps.map(app => `
+                <div class="app-compact" onclick="marketplace.showAppDetail(${JSON.stringify(app).replace(/"/g, '&quot;')})">
+                    <div class="app-compact-header">
+                        <span>${app.category}</span>
+                        <span>${app.type}</span>
+                    </div>
+                    <div class="app-compact-title">${app.name}</div>
+                </div>
+            `).join('');
+        }
+    }
+
+    setupEventListeners() {
+        // Search
+        const searchInput = document.getElementById('search-input');
+        searchInput.addEventListener('input', (e) => {
+            clearTimeout(this.searchTimeout);
+            this.searchTimeout = setTimeout(() => this.search(e.target.value), 300);
+        });
+
+        // Keyboard shortcut
+        document.addEventListener('keydown', (e) => {
+            if (e.key === '/' && !searchInput.contains(document.activeElement)) {
+                e.preventDefault();
+                searchInput.focus();
+            }
+            if (e.key === 'Escape' && searchInput.contains(document.activeElement)) {
+                searchInput.blur();
+                searchInput.value = '';
+            }
+        });
+
+        // Type filter
+        const typeFilter = document.getElementById('type-filter');
+        typeFilter.addEventListener('change', (e) => {
+            this.currentType = e.target.value;
+            this.loadMainContent();
+        });
+
+        // Load more
+        const loadMore = document.getElementById('load-more');
+        loadMore.addEventListener('click', () => this.loadMoreApps());
+    }
+
+    async filterByCategory(category) {
+        // Update active state
+        document.querySelectorAll('.filter-btn').forEach(btn => {
+            btn.classList.toggle('active', btn.dataset.category === category);
+        });
+
+        this.currentCategory = category;
+        await this.loadMainContent();
+    }
+
+    async search(query) {
+        if (!query) {
+            await this.loadMainContent();
+            return;
+        }
+
+        const results = await this.api.search(query);
+        if (!results) return;
+
+        // Update apps grid with search results
+        if (results.apps && results.apps.length) {
+            const appsGrid = document.getElementById('apps-grid');
+            appsGrid.innerHTML = results.apps.map(app => `
+                <div class="app-compact" onclick="marketplace.showAppDetail(${JSON.stringify(app).replace(/"/g, '&quot;')})">
+                    <div class="app-compact-header">
+                        <span>${app.category}</span>
+                        <span>★ ${app.rating}/5</span>
+                    </div>
+                    <div class="app-compact-title">${app.name}</div>
+                    <div class="app-compact-desc">${app.description}</div>
+                </div>
+            `).join('');
+        }
+
+        // Update articles with search results
+        if (results.articles && results.articles.length) {
+            const articlesList = document.getElementById('articles-list');
+            articlesList.innerHTML = results.articles.map(article => `
+                <div class="article-compact" onclick="marketplace.showArticle('${article.id}')">
+                    <div class="article-meta">
+                        <span>${article.category}</span> · <span>${new Date(article.published_at).toLocaleDateString()}</span>
+                    </div>
+                    <div class="article-title">${article.title}</div>
+                    <div class="article-author">by ${article.author}</div>
+                </div>
+            `).join('');
+        }
+    }
+
+    async loadMoreApps() {
+        this.loadedApps += 12;
+        const moreApps = await this.api.getApps({ offset: this.loadedApps, limit: 12 });
+        if (moreApps && moreApps.length) {
+            const moreGrid = document.getElementById('more-apps-grid');
+            moreApps.forEach(app => {
+                const card = document.createElement('div');
+                card.className = 'app-compact';
+                card.innerHTML = `
+                    <div class="app-compact-header">
+                        <span>${app.category}</span>
+                        <span>${app.type}</span>
+                    </div>
+                    <div class="app-compact-title">${app.name}</div>
+                `;
+                card.onclick = () => this.showAppDetail(app);
+                moreGrid.appendChild(card);
+            });
+        }
+    }
+
+    showAppDetail(app) {
+        // Navigate to detail page instead of showing modal
+        const slug = app.slug || app.name.toLowerCase().replace(/\s+/g, '-');
+        window.location.href = `app-detail.html?app=${slug}`;
+    }
+
+    showArticle(articleId) {
+        // Could create article detail page similarly
+        console.log('Show article:', articleId);
+    }
+}
+
+// Initialize marketplace
+let marketplace;
+document.addEventListener('DOMContentLoaded', () => {
+    marketplace = new MarketplaceUI();
+});

mkdocs.yml

@@ -1,5 +1,4 @@
 site_name: Crawl4AI Documentation (v0.7.x)
-site_favicon: docs/md_v2/favicon.ico
 site_description:  🚀🤖 Crawl4AI, Open-source LLM-Friendly Web Crawler & Scraper
 site_url: https://docs.crawl4ai.com
 repo_url: https://github.com/unclecode/crawl4ai
@@ -8,6 +7,7 @@ docs_dir: docs/md_v2
 
 nav:
   - Home: 'index.md'
+  - "📚 Complete SDK Reference": "complete-sdk-reference.md"
   - "Ask AI": "core/ask-ai.md"
   - "Quick Start": "core/quickstart.md"
   - "Code Examples": "core/examples.md"
@@ -15,6 +15,8 @@ nav:
     - "Demo Apps": "apps/index.md"
     - "C4A-Script Editor": "apps/c4a-script/index.html"
     - "LLM Context Builder": "apps/llmtxt/index.html"
+    - "Marketplace": "marketplace/index.html"
+    - "Marketplace Admin": "marketplace/admin/index.html"
   - Setup & Installation:
     - "Installation": "core/installation.md"
     - "Docker Deployment": "core/docker-deployment.md"
@@ -59,7 +61,6 @@ nav:
     - "Clustering Strategies": "extraction/clustring-strategies.md"
     - "Chunking": "extraction/chunking.md"
   - API Reference:
-    - "Docker Server API": "api/docker-server.md"
     - "AsyncWebCrawler": "api/async-webcrawler.md"
     - "arun()": "api/arun.md"
     - "arun_many()": "api/arun_many.md"
@@ -67,10 +68,12 @@ nav:
     - "CrawlResult": "api/crawl-result.md"
     - "Strategies": "api/strategies.md"
     - "C4A-Script Reference": "api/c4a-script-reference.md"
+  - "Brand Book": "branding/index.md"
 
 theme:
   name: 'terminal'
   palette: 'dark'
+  favicon: favicon.ico
   custom_dir: docs/md_v2/overrides
   color_mode: 'dark'
   icon:
@@ -99,6 +102,7 @@ extra_css:
   - assets/highlight.css
   - assets/dmvendor.css
   - assets/feedback-overrides.css
+  - assets/page_actions.css
 
 extra_javascript:
   - https://www.googletagmanager.com/gtag/js?id=G-58W0K2ZQ25
@@ -107,8 +111,9 @@ extra_javascript:
   - assets/highlight_init.js
   - https://buttons.github.io/buttons.js
   - assets/toc.js
-  - assets/github_stats.js 
+  - assets/github_stats.js
   - assets/selection_ask_ai.js
   - assets/copy_code.js
   - assets/floating_ask_ai_button.js
-  - assets/mobile_menu.js
+  - assets/mobile_menu.js
+  - assets/page_actions.js?v=20251006

tests/docker/extended_features/demo_adaptive_endpoint.py

@@ -1,435 +0,0 @@
-#!/usr/bin/env python3
-"""
-Demo: How users will call the Adaptive Digest endpoint
-This shows practical examples of how developers would use the adaptive crawling
-feature to intelligently gather relevant content based on queries.
-"""
-
-import asyncio
-import time
-from typing import Any, Dict, Optional
-
-import aiohttp
-
-# Configuration
-API_BASE_URL = "http://localhost:11235"
-API_TOKEN = None  # Set if your API requires authentication
-
-
-class AdaptiveEndpointDemo:
-    def __init__(self, base_url: str = API_BASE_URL, token: str = None):
-        self.base_url = base_url
-        self.headers = {"Content-Type": "application/json"}
-        if token:
-            self.headers["Authorization"] = f"Bearer {token}"
-
-    async def submit_adaptive_job(
-        self, start_url: str, query: str, config: Optional[Dict] = None
-    ) -> str:
-        """Submit an adaptive crawling job and return task ID"""
-        payload = {"start_url": start_url, "query": query}
-
-        if config:
-            payload["config"] = config
-
-        async with aiohttp.ClientSession() as session:
-            async with session.post(
-                f"{self.base_url}/adaptive/digest/job",
-                headers=self.headers,
-                json=payload,
-            ) as response:
-                if response.status == 202:  # Accepted
-                    result = await response.json()
-                    return result["task_id"]
-                else:
-                    error_text = await response.text()
-                    raise Exception(f"API Error {response.status}: {error_text}")
-
-    async def check_job_status(self, task_id: str) -> Dict[str, Any]:
-        """Check the status of an adaptive crawling job"""
-        async with aiohttp.ClientSession() as session:
-            async with session.get(
-                f"{self.base_url}/adaptive/digest/job/{task_id}", headers=self.headers
-            ) as response:
-                if response.status == 200:
-                    return await response.json()
-                else:
-                    error_text = await response.text()
-                    raise Exception(f"API Error {response.status}: {error_text}")
-
-    async def wait_for_completion(
-        self, task_id: str, max_wait: int = 300
-    ) -> Dict[str, Any]:
-        """Poll job status until completion or timeout"""
-        start_time = time.time()
-
-        while time.time() - start_time < max_wait:
-            status = await self.check_job_status(task_id)
-
-            if status["status"] == "COMPLETED":
-                return status
-            elif status["status"] == "FAILED":
-                raise Exception(f"Job failed: {status.get('error', 'Unknown error')}")
-
-            print(
-                f"⏳ Job {status['status']}... (elapsed: {int(time.time() - start_time)}s)"
-            )
-            await asyncio.sleep(3)  # Poll every 3 seconds
-
-        raise Exception(f"Job timed out after {max_wait} seconds")
-
-    async def demo_research_assistant(self):
-        """Demo: Research assistant for academic papers"""
-        print("🔬 Demo: Academic Research Assistant")
-        print("=" * 50)
-
-        try:
-            print("🚀 Submitting job: Find research on 'machine learning optimization'")
-
-            task_id = await self.submit_adaptive_job(
-                start_url="https://arxiv.org",
-                query="machine learning optimization techniques recent papers",
-                config={
-                    "max_depth": 3,
-                    "confidence_threshold": 0.7,
-                    "max_pages": 20,
-                    "content_filters": ["academic", "research"],
-                },
-            )
-
-            print(f"📋 Job submitted with ID: {task_id}")
-
-            # Wait for completion
-            result = await self.wait_for_completion(task_id)
-
-            print("✅ Research completed!")
-            print(f"🎯 Confidence score: {result['result']['confidence']:.2f}")
-            print(f"📊 Coverage stats: {result['result']['coverage_stats']}")
-
-            # Show relevant content found
-            relevant_content = result["result"]["relevant_content"]
-            print(f"\n📚 Found {len(relevant_content)} relevant research papers:")
-
-            for i, content in enumerate(relevant_content[:3], 1):
-                title = content.get("title", "Untitled")[:60]
-                relevance = content.get("relevance_score", 0)
-                print(f"  {i}. {title}... (relevance: {relevance:.2f})")
-
-        except Exception as e:
-            print(f"❌ Error: {e}")
-
-    async def demo_market_intelligence(self):
-        """Demo: Market intelligence gathering"""
-        print("\n💼 Demo: Market Intelligence Gathering")
-        print("=" * 50)
-
-        try:
-            print("🚀 Submitting job: Analyze competitors in 'sustainable packaging'")
-
-            task_id = await self.submit_adaptive_job(
-                start_url="https://packagingeurope.com",
-                query="sustainable packaging solutions eco-friendly materials competitors market trends",
-                config={
-                    "max_depth": 4,
-                    "confidence_threshold": 0.6,
-                    "max_pages": 30,
-                    "content_filters": ["business", "industry"],
-                    "follow_external_links": True,
-                },
-            )
-
-            print(f"📋 Job submitted with ID: {task_id}")
-
-            # Wait for completion
-            result = await self.wait_for_completion(task_id)
-
-            print("✅ Market analysis completed!")
-            print(f"🎯 Intelligence confidence: {result['result']['confidence']:.2f}")
-
-            # Analyze findings
-            relevant_content = result["result"]["relevant_content"]
-            print(
-                f"\n📈 Market intelligence gathered from {len(relevant_content)} sources:"
-            )
-
-            companies = set()
-            trends = []
-
-            for content in relevant_content:
-                # Extract company mentions (simplified)
-                text = content.get("content", "")
-                if any(
-                    word in text.lower()
-                    for word in ["company", "corporation", "inc", "ltd"]
-                ):
-                    # This would be more sophisticated in real implementation
-                    companies.add(content.get("source_url", "Unknown"))
-
-                # Extract trend keywords
-                if any(
-                    word in text.lower() for word in ["trend", "innovation", "future"]
-                ):
-                    trends.append(content.get("title", "Trend"))
-
-            print(f"🏢 Companies analyzed: {len(companies)}")
-            print(f"📊 Trends identified: {len(trends)}")
-
-        except Exception as e:
-            print(f"❌ Error: {e}")
-
-    async def demo_content_curation(self):
-        """Demo: Content curation for newsletter"""
-        print("\n📰 Demo: Content Curation for Tech Newsletter")
-        print("=" * 50)
-
-        try:
-            print("🚀 Submitting job: Curate content about 'AI developments this week'")
-
-            task_id = await self.submit_adaptive_job(
-                start_url="https://techcrunch.com",
-                query="artificial intelligence AI developments news this week recent advances",
-                config={
-                    "max_depth": 2,
-                    "confidence_threshold": 0.8,
-                    "max_pages": 25,
-                    "content_filters": ["news", "recent"],
-                    "date_range": "last_7_days",
-                },
-            )
-
-            print(f"📋 Job submitted with ID: {task_id}")
-
-            # Wait for completion
-            result = await self.wait_for_completion(task_id)
-
-            print("✅ Content curation completed!")
-            print(f"🎯 Curation confidence: {result['result']['confidence']:.2f}")
-
-            # Process curated content
-            relevant_content = result["result"]["relevant_content"]
-            print(f"\n📮 Curated {len(relevant_content)} articles for your newsletter:")
-
-            # Group by category/topic
-            categories = {
-                "AI Research": [],
-                "Industry News": [],
-                "Product Launches": [],
-                "Other": [],
-            }
-
-            for content in relevant_content:
-                title = content.get("title", "Untitled")
-                if any(
-                    word in title.lower() for word in ["research", "study", "paper"]
-                ):
-                    categories["AI Research"].append(content)
-                elif any(
-                    word in title.lower() for word in ["company", "startup", "funding"]
-                ):
-                    categories["Industry News"].append(content)
-                elif any(
-                    word in title.lower() for word in ["launch", "release", "unveil"]
-                ):
-                    categories["Product Launches"].append(content)
-                else:
-                    categories["Other"].append(content)
-
-            for category, articles in categories.items():
-                if articles:
-                    print(f"\n📂 {category} ({len(articles)} articles):")
-                    for article in articles[:2]:  # Show top 2 per category
-                        title = article.get("title", "Untitled")[:50]
-                        print(f"  • {title}...")
-
-        except Exception as e:
-            print(f"❌ Error: {e}")
-
-    async def demo_product_research(self):
-        """Demo: Product research and comparison"""
-        print("\n🛍️ Demo: Product Research & Comparison")
-        print("=" * 50)
-
-        try:
-            print("🚀 Submitting job: Research 'best wireless headphones 2024'")
-
-            task_id = await self.submit_adaptive_job(
-                start_url="https://www.cnet.com",
-                query="best wireless headphones 2024 reviews comparison features price",
-                config={
-                    "max_depth": 3,
-                    "confidence_threshold": 0.75,
-                    "max_pages": 20,
-                    "content_filters": ["review", "comparison"],
-                    "extract_structured_data": True,
-                },
-            )
-
-            print(f"📋 Job submitted with ID: {task_id}")
-
-            # Wait for completion
-            result = await self.wait_for_completion(task_id)
-
-            print("✅ Product research completed!")
-            print(f"🎯 Research confidence: {result['result']['confidence']:.2f}")
-
-            # Analyze product data
-            relevant_content = result["result"]["relevant_content"]
-            print(
-                f"\n🎧 Product research summary from {len(relevant_content)} sources:"
-            )
-
-            # Extract product mentions (simplified example)
-            products = {}
-            for content in relevant_content:
-                text = content.get("content", "").lower()
-                # Look for common headphone brands
-                brands = [
-                    "sony",
-                    "bose",
-                    "apple",
-                    "sennheiser",
-                    "jabra",
-                    "audio-technica",
-                ]
-                for brand in brands:
-                    if brand in text:
-                        if brand not in products:
-                            products[brand] = 0
-                        products[brand] += 1
-
-            print("🏷️ Product mentions:")
-            for product, mentions in sorted(
-                products.items(), key=lambda x: x[1], reverse=True
-            )[:5]:
-                print(f"  {product.title()}: {mentions} mentions")
-
-        except Exception as e:
-            print(f"❌ Error: {e}")
-
-    async def demo_monitoring_pipeline(self):
-        """Demo: Set up a monitoring pipeline for ongoing content tracking"""
-        print("\n📡 Demo: Content Monitoring Pipeline")
-        print("=" * 50)
-
-        monitoring_queries = [
-            {
-                "name": "Brand Mentions",
-                "start_url": "https://news.google.com",
-                "query": "YourBrand company news mentions",
-                "priority": "high",
-            },
-            {
-                "name": "Industry Trends",
-                "start_url": "https://techcrunch.com",
-                "query": "SaaS industry trends 2024",
-                "priority": "medium",
-            },
-            {
-                "name": "Competitor Activity",
-                "start_url": "https://crunchbase.com",
-                "query": "competitor funding announcements product launches",
-                "priority": "high",
-            },
-        ]
-
-        print("🚀 Starting monitoring pipeline with 3 queries...")
-
-        jobs = {}
-
-        # Submit all monitoring jobs
-        for query_config in monitoring_queries:
-            print(f"\n📋 Submitting: {query_config['name']}")
-
-            try:
-                task_id = await self.submit_adaptive_job(
-                    start_url=query_config["start_url"],
-                    query=query_config["query"],
-                    config={
-                        "max_depth": 2,
-                        "confidence_threshold": 0.6,
-                        "max_pages": 15,
-                    },
-                )
-
-                jobs[query_config["name"]] = {
-                    "task_id": task_id,
-                    "priority": query_config["priority"],
-                    "status": "submitted",
-                }
-
-                print(f"  ✅ Job ID: {task_id}")
-
-            except Exception as e:
-                print(f"  ❌ Failed: {e}")
-
-        # Monitor all jobs
-        print(f"\n⏳ Monitoring {len(jobs)} jobs...")
-
-        completed_jobs = {}
-        max_wait = 180  # 3 minutes total
-        start_time = time.time()
-
-        while jobs and (time.time() - start_time) < max_wait:
-            for name, job_info in list(jobs.items()):
-                try:
-                    status = await self.check_job_status(job_info["task_id"])
-
-                    if status["status"] == "COMPLETED":
-                        completed_jobs[name] = status
-                        del jobs[name]
-                        print(f"  ✅ {name} completed")
-                    elif status["status"] == "FAILED":
-                        print(f"  ❌ {name} failed: {status.get('error', 'Unknown')}")
-                        del jobs[name]
-
-                except Exception as e:
-                    print(f"  ⚠️ Error checking {name}: {e}")
-
-            if jobs:  # Still have pending jobs
-                await asyncio.sleep(5)
-
-        # Summary
-        print("\n📊 Monitoring Pipeline Summary:")
-        print(f"  ✅ Completed: {len(completed_jobs)} jobs")
-        print(f"  ⏳ Pending: {len(jobs)} jobs")
-
-        for name, result in completed_jobs.items():
-            confidence = result["result"]["confidence"]
-            content_count = len(result["result"]["relevant_content"])
-            print(f"    {name}: {content_count} items (confidence: {confidence:.2f})")
-
-
-async def main():
-    """Run all adaptive endpoint demos"""
-    print("🧠 Crawl4AI Adaptive Digest Endpoint - User Demo")
-    print("=" * 60)
-    print("This demo shows how developers use adaptive crawling")
-    print("to intelligently gather relevant content based on queries.\n")
-
-    demo = AdaptiveEndpointDemo()
-
-    try:
-        # Run individual demos
-        await demo.demo_research_assistant()
-        await demo.demo_market_intelligence()
-        await demo.demo_content_curation()
-        await demo.demo_product_research()
-
-        # Run monitoring pipeline demo
-        await demo.demo_monitoring_pipeline()
-
-        print("\n🎉 All demos completed successfully!")
-        print("\nReal-world usage patterns:")
-        print("1. Submit multiple jobs for parallel processing")
-        print("2. Poll job status to track progress")
-        print("3. Process results when jobs complete")
-        print("4. Use confidence scores to filter quality content")
-        print("5. Set up monitoring pipelines for ongoing intelligence")
-
-    except Exception as e:
-        print(f"\n❌ Demo failed: {e}")
-        print("Make sure the Crawl4AI server is running on localhost:11235")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

tests/docker/extended_features/demo_monitoring_dashboard.py

@@ -1,479 +0,0 @@
-"""
-Interactive Monitoring Dashboard Demo
-
-This demo showcases the monitoring and profiling capabilities of Crawl4AI's Docker server.
-It provides:
-- Real-time statistics dashboard with auto-refresh
-- Profiling session management
-- System resource monitoring
-- URL-specific statistics
-- Interactive terminal UI
-
-Usage:
-    python demo_monitoring_dashboard.py [--url BASE_URL]
-"""
-
-import argparse
-import asyncio
-import json
-import sys
-import time
-from datetime import datetime
-from typing import Dict, List, Optional
-
-import httpx
-
-
-class Colors:
-    """ANSI color codes for terminal output."""
-    HEADER = '\033[95m'
-    OKBLUE = '\033[94m'
-    OKCYAN = '\033[96m'
-    OKGREEN = '\033[92m'
-    WARNING = '\033[93m'
-    FAIL = '\033[91m'
-    ENDC = '\033[0m'
-    BOLD = '\033[1m'
-    UNDERLINE = '\033[4m'
-
-
-class MonitoringDashboard:
-    """Interactive monitoring dashboard for Crawl4AI."""
-
-    def __init__(self, base_url: str = "http://localhost:11234"):
-        self.base_url = base_url
-        self.client = httpx.AsyncClient(base_url=base_url, timeout=60.0)
-        self.running = True
-        self.current_view = "dashboard"  # dashboard, sessions, urls
-        self.profiling_sessions: List[Dict] = []
-
-    async def close(self):
-        """Close the HTTP client."""
-        await self.client.aclose()
-
-    def clear_screen(self):
-        """Clear the terminal screen."""
-        print("\033[2J\033[H", end="")
-
-    def print_header(self, title: str):
-        """Print a formatted header."""
-        width = 80
-        print(f"\n{Colors.HEADER}{Colors.BOLD}")
-        print("=" * width)
-        print(f"{title.center(width)}")
-        print("=" * width)
-        print(f"{Colors.ENDC}")
-
-    def print_section(self, title: str):
-        """Print a section header."""
-        print(f"\n{Colors.OKBLUE}{Colors.BOLD}▶ {title}{Colors.ENDC}")
-        print("-" * 80)
-
-    async def check_health(self) -> Dict:
-        """Check server health."""
-        try:
-            response = await self.client.get("/monitoring/health")
-            response.raise_for_status()
-            return response.json()
-        except Exception as e:
-            return {"status": "error", "error": str(e)}
-
-    async def get_stats(self) -> Dict:
-        """Get current statistics."""
-        try:
-            response = await self.client.get("/monitoring/stats")
-            response.raise_for_status()
-            return response.json()
-        except Exception as e:
-            return {"error": str(e)}
-
-    async def get_url_stats(self) -> List[Dict]:
-        """Get URL-specific statistics."""
-        try:
-            response = await self.client.get("/monitoring/stats/urls")
-            response.raise_for_status()
-            return response.json()
-        except Exception as e:
-            return []
-
-    async def list_profiling_sessions(self) -> List[Dict]:
-        """List all profiling sessions."""
-        try:
-            response = await self.client.get("/monitoring/profile")
-            response.raise_for_status()
-            data = response.json()
-            return data.get("sessions", [])
-        except Exception as e:
-            return []
-
-    async def start_profiling_session(self, urls: List[str], duration: int = 30) -> Dict:
-        """Start a new profiling session."""
-        try:
-            request_data = {
-                "urls": urls,
-                "duration_seconds": duration,
-                "crawler_config": {
-                    "word_count_threshold": 10
-                }
-            }
-            response = await self.client.post("/monitoring/profile/start", json=request_data)
-            response.raise_for_status()
-            return response.json()
-        except Exception as e:
-            return {"error": str(e)}
-
-    async def get_profiling_session(self, session_id: str) -> Dict:
-        """Get profiling session details."""
-        try:
-            response = await self.client.get(f"/monitoring/profile/{session_id}")
-            response.raise_for_status()
-            return response.json()
-        except Exception as e:
-            return {"error": str(e)}
-
-    async def delete_profiling_session(self, session_id: str) -> Dict:
-        """Delete a profiling session."""
-        try:
-            response = await self.client.delete(f"/monitoring/profile/{session_id}")
-            response.raise_for_status()
-            return response.json()
-        except Exception as e:
-            return {"error": str(e)}
-
-    async def reset_stats(self) -> Dict:
-        """Reset all statistics."""
-        try:
-            response = await self.client.post("/monitoring/stats/reset")
-            response.raise_for_status()
-            return response.json()
-        except Exception as e:
-            return {"error": str(e)}
-
-    def display_dashboard(self, stats: Dict):
-        """Display the main statistics dashboard."""
-        self.clear_screen()
-        self.print_header("Crawl4AI Monitoring Dashboard")
-
-        # Health Status
-        print(f"\n{Colors.OKGREEN}● Server Status: ONLINE{Colors.ENDC}")
-        print(f"Base URL: {self.base_url}")
-        print(f"Last Updated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
-
-        # Crawler Statistics
-        self.print_section("Crawler Statistics")
-        if "error" in stats:
-            print(f"{Colors.FAIL}Error fetching stats: {stats['error']}{Colors.ENDC}")
-        else:
-            print(f"Active Crawls:      {Colors.BOLD}{stats.get('active_crawls', 0)}{Colors.ENDC}")
-            print(f"Total Crawls:       {stats.get('total_crawls', 0)}")
-            print(f"Successful:         {Colors.OKGREEN}{stats.get('successful_crawls', 0)}{Colors.ENDC}")
-            print(f"Failed:             {Colors.FAIL}{stats.get('failed_crawls', 0)}{Colors.ENDC}")
-            print(f"Success Rate:       {stats.get('success_rate', 0):.2f}%")
-            print(f"Avg Duration:       {stats.get('avg_duration_ms', 0):.2f} ms")
-            
-            # Format bytes
-            total_bytes = stats.get('total_bytes_processed', 0)
-            if total_bytes > 1024 * 1024:
-                bytes_str = f"{total_bytes / (1024 * 1024):.2f} MB"
-            elif total_bytes > 1024:
-                bytes_str = f"{total_bytes / 1024:.2f} KB"
-            else:
-                bytes_str = f"{total_bytes} bytes"
-            print(f"Total Data Processed: {bytes_str}")
-
-        # System Statistics
-        if "system_stats" in stats:
-            self.print_section("System Resources")
-            sys_stats = stats["system_stats"]
-            
-            cpu = sys_stats.get("cpu_percent", 0)
-            cpu_color = Colors.OKGREEN if cpu < 50 else Colors.WARNING if cpu < 80 else Colors.FAIL
-            print(f"CPU Usage:          {cpu_color}{cpu:.1f}%{Colors.ENDC}")
-            
-            mem = sys_stats.get("memory_percent", 0)
-            mem_color = Colors.OKGREEN if mem < 50 else Colors.WARNING if mem < 80 else Colors.FAIL
-            print(f"Memory Usage:       {mem_color}{mem:.1f}%{Colors.ENDC}")
-            
-            mem_used = sys_stats.get("memory_used_mb", 0)
-            mem_available = sys_stats.get("memory_available_mb", 0)
-            print(f"Memory Used:        {mem_used:.0f} MB / {mem_available:.0f} MB")
-            
-            disk = sys_stats.get("disk_usage_percent", 0)
-            disk_color = Colors.OKGREEN if disk < 70 else Colors.WARNING if disk < 90 else Colors.FAIL
-            print(f"Disk Usage:         {disk_color}{disk:.1f}%{Colors.ENDC}")
-            
-            print(f"Active Processes:   {sys_stats.get('active_processes', 0)}")
-
-        # Navigation
-        self.print_section("Navigation")
-        print(f"[D] Dashboard  [S] Profiling Sessions  [U] URL Stats  [R] Reset Stats  [Q] Quit")
-
-    def display_url_stats(self, url_stats: List[Dict]):
-        """Display URL-specific statistics."""
-        self.clear_screen()
-        self.print_header("URL Statistics")
-
-        if not url_stats:
-            print(f"\n{Colors.WARNING}No URL statistics available yet.{Colors.ENDC}")
-        else:
-            print(f"\nTotal URLs tracked: {len(url_stats)}")
-            print()
-            
-            # Table header
-            print(f"{Colors.BOLD}{'URL':<50} {'Requests':<10} {'Success':<10} {'Avg Time':<12} {'Data':<12}{Colors.ENDC}")
-            print("-" * 94)
-            
-            # Sort by total requests
-            sorted_stats = sorted(url_stats, key=lambda x: x.get('total_requests', 0), reverse=True)
-            
-            for stat in sorted_stats[:20]:  # Show top 20
-                url = stat.get('url', 'unknown')
-                if len(url) > 47:
-                    url = url[:44] + "..."
-                
-                total = stat.get('total_requests', 0)
-                success = stat.get('successful_requests', 0)
-                success_pct = f"{(success/total*100):.0f}%" if total > 0 else "N/A"
-                
-                avg_time = stat.get('avg_duration_ms', 0)
-                time_str = f"{avg_time:.0f} ms"
-                
-                bytes_processed = stat.get('total_bytes_processed', 0)
-                if bytes_processed > 1024 * 1024:
-                    data_str = f"{bytes_processed / (1024 * 1024):.2f} MB"
-                elif bytes_processed > 1024:
-                    data_str = f"{bytes_processed / 1024:.2f} KB"
-                else:
-                    data_str = f"{bytes_processed} B"
-                
-                print(f"{url:<50} {total:<10} {success_pct:<10} {time_str:<12} {data_str:<12}")
-
-        # Navigation
-        self.print_section("Navigation")
-        print(f"[D] Dashboard  [S] Profiling Sessions  [U] URL Stats  [R] Reset Stats  [Q] Quit")
-
-    def display_profiling_sessions(self, sessions: List[Dict]):
-        """Display profiling sessions."""
-        self.clear_screen()
-        self.print_header("Profiling Sessions")
-
-        if not sessions:
-            print(f"\n{Colors.WARNING}No profiling sessions found.{Colors.ENDC}")
-        else:
-            print(f"\nTotal sessions: {len(sessions)}")
-            print()
-            
-            # Table header
-            print(f"{Colors.BOLD}{'ID':<25} {'Status':<12} {'URLs':<6} {'Duration':<12} {'Started':<20}{Colors.ENDC}")
-            print("-" * 85)
-            
-            # Sort by started time (newest first)
-            sorted_sessions = sorted(sessions, key=lambda x: x.get('started_at', ''), reverse=True)
-            
-            for session in sorted_sessions[:15]:  # Show top 15
-                session_id = session.get('session_id', 'unknown')
-                if len(session_id) > 22:
-                    session_id = session_id[:19] + "..."
-                
-                status = session.get('status', 'unknown')
-                status_color = Colors.OKGREEN if status == 'completed' else Colors.WARNING if status == 'running' else Colors.FAIL
-                
-                url_count = len(session.get('urls', []))
-                
-                duration = session.get('duration_seconds', 0)
-                duration_str = f"{duration}s" if duration else "N/A"
-                
-                started = session.get('started_at', 'N/A')
-                if started != 'N/A':
-                    try:
-                        dt = datetime.fromisoformat(started.replace('Z', '+00:00'))
-                        started = dt.strftime('%Y-%m-%d %H:%M:%S')
-                    except:
-                        pass
-                
-                print(f"{session_id:<25} {status_color}{status:<12}{Colors.ENDC} {url_count:<6} {duration_str:<12} {started:<20}")
-
-        # Navigation
-        self.print_section("Navigation & Actions")
-        print(f"[D] Dashboard  [S] Profiling Sessions  [U] URL Stats")
-        print(f"[N] New Session  [V] View Session  [X] Delete Session")
-        print(f"[R] Reset Stats  [Q] Quit")
-
-    async def interactive_session_view(self, session_id: str):
-        """Display detailed view of a profiling session."""
-        session = await self.get_profiling_session(session_id)
-        
-        self.clear_screen()
-        self.print_header(f"Profiling Session: {session_id}")
-        
-        if "error" in session:
-            print(f"\n{Colors.FAIL}Error: {session['error']}{Colors.ENDC}")
-        else:
-            print(f"\n{Colors.BOLD}Session ID:{Colors.ENDC} {session.get('session_id', 'N/A')}")
-            
-            status = session.get('status', 'unknown')
-            status_color = Colors.OKGREEN if status == 'completed' else Colors.WARNING
-            print(f"{Colors.BOLD}Status:{Colors.ENDC} {status_color}{status}{Colors.ENDC}")
-            
-            print(f"{Colors.BOLD}URLs:{Colors.ENDC}")
-            for url in session.get('urls', []):
-                print(f"  - {url}")
-            
-            started = session.get('started_at', 'N/A')
-            print(f"{Colors.BOLD}Started:{Colors.ENDC} {started}")
-            
-            if 'completed_at' in session:
-                print(f"{Colors.BOLD}Completed:{Colors.ENDC} {session['completed_at']}")
-            
-            if 'results' in session:
-                self.print_section("Profiling Results")
-                results = session['results']
-                
-                print(f"Total Requests:     {results.get('total_requests', 0)}")
-                print(f"Successful:         {Colors.OKGREEN}{results.get('successful_requests', 0)}{Colors.ENDC}")
-                print(f"Failed:             {Colors.FAIL}{results.get('failed_requests', 0)}{Colors.ENDC}")
-                print(f"Avg Response Time:  {results.get('avg_response_time_ms', 0):.2f} ms")
-                
-                if 'system_metrics' in results:
-                    self.print_section("System Metrics During Profiling")
-                    metrics = results['system_metrics']
-                    print(f"Avg CPU:            {metrics.get('avg_cpu_percent', 0):.1f}%")
-                    print(f"Peak CPU:           {metrics.get('peak_cpu_percent', 0):.1f}%")
-                    print(f"Avg Memory:         {metrics.get('avg_memory_percent', 0):.1f}%")
-                    print(f"Peak Memory:        {metrics.get('peak_memory_percent', 0):.1f}%")
-        
-        print(f"\n{Colors.OKCYAN}Press any key to return...{Colors.ENDC}")
-        input()
-
-    async def create_new_session(self):
-        """Interactive session creation."""
-        self.clear_screen()
-        self.print_header("Create New Profiling Session")
-        
-        print(f"\n{Colors.BOLD}Enter URLs to profile (one per line, empty line to finish):{Colors.ENDC}")
-        urls = []
-        while True:
-            url = input(f"{Colors.OKCYAN}URL {len(urls) + 1}:{Colors.ENDC} ").strip()
-            if not url:
-                break
-            urls.append(url)
-        
-        if not urls:
-            print(f"{Colors.FAIL}No URLs provided. Cancelled.{Colors.ENDC}")
-            time.sleep(2)
-            return
-        
-        duration = input(f"{Colors.OKCYAN}Duration (seconds, default 30):{Colors.ENDC} ").strip()
-        try:
-            duration = int(duration) if duration else 30
-        except:
-            duration = 30
-        
-        print(f"\n{Colors.WARNING}Starting profiling session for {len(urls)} URL(s), {duration}s...{Colors.ENDC}")
-        result = await self.start_profiling_session(urls, duration)
-        
-        if "error" in result:
-            print(f"{Colors.FAIL}Error: {result['error']}{Colors.ENDC}")
-        else:
-            print(f"{Colors.OKGREEN}✓ Session started successfully!{Colors.ENDC}")
-            print(f"Session ID: {result.get('session_id', 'N/A')}")
-        
-        time.sleep(3)
-
-    async def run_dashboard(self):
-        """Run the interactive dashboard."""
-        print(f"{Colors.OKGREEN}Starting Crawl4AI Monitoring Dashboard...{Colors.ENDC}")
-        print(f"Connecting to {self.base_url}...")
-        
-        # Check health
-        health = await self.check_health()
-        if health.get("status") != "healthy":
-            print(f"{Colors.FAIL}Error: Server not responding or unhealthy{Colors.ENDC}")
-            print(f"Health check result: {health}")
-            return
-        
-        print(f"{Colors.OKGREEN}✓ Connected successfully!{Colors.ENDC}")
-        time.sleep(1)
-        
-        # Main loop
-        while self.running:
-            if self.current_view == "dashboard":
-                stats = await self.get_stats()
-                self.display_dashboard(stats)
-            elif self.current_view == "urls":
-                url_stats = await self.get_url_stats()
-                self.display_url_stats(url_stats)
-            elif self.current_view == "sessions":
-                sessions = await self.list_profiling_sessions()
-                self.display_profiling_sessions(sessions)
-            
-            # Get user input (non-blocking with timeout)
-            print(f"\n{Colors.OKCYAN}Enter command (or wait 5s for auto-refresh):{Colors.ENDC} ", end="", flush=True)
-            
-            try:
-                # Simple input with timeout simulation
-                import select
-                if sys.platform != 'win32':
-                    i, _, _ = select.select([sys.stdin], [], [], 5.0)
-                    if i:
-                        command = sys.stdin.readline().strip().lower()
-                    else:
-                        command = ""
-                else:
-                    # Windows doesn't support select on stdin
-                    command = input()
-            except:
-                command = ""
-            
-            # Process command
-            if command == 'q':
-                self.running = False
-            elif command == 'd':
-                self.current_view = "dashboard"
-            elif command == 's':
-                self.current_view = "sessions"
-            elif command == 'u':
-                self.current_view = "urls"
-            elif command == 'r':
-                print(f"\n{Colors.WARNING}Resetting statistics...{Colors.ENDC}")
-                await self.reset_stats()
-                time.sleep(1)
-            elif command == 'n' and self.current_view == "sessions":
-                await self.create_new_session()
-            elif command == 'v' and self.current_view == "sessions":
-                session_id = input(f"{Colors.OKCYAN}Enter session ID:{Colors.ENDC} ").strip()
-                if session_id:
-                    await self.interactive_session_view(session_id)
-            elif command == 'x' and self.current_view == "sessions":
-                session_id = input(f"{Colors.OKCYAN}Enter session ID to delete:{Colors.ENDC} ").strip()
-                if session_id:
-                    result = await self.delete_profiling_session(session_id)
-                    if "error" in result:
-                        print(f"{Colors.FAIL}Error: {result['error']}{Colors.ENDC}")
-                    else:
-                        print(f"{Colors.OKGREEN}✓ Session deleted{Colors.ENDC}")
-                    time.sleep(2)
-        
-        self.clear_screen()
-        print(f"\n{Colors.OKGREEN}Dashboard closed. Goodbye!{Colors.ENDC}\n")
-
-
-async def main():
-    """Main entry point."""
-    parser = argparse.ArgumentParser(description="Crawl4AI Monitoring Dashboard")
-    parser.add_argument(
-        "--url",
-        default="http://localhost:11234",
-        help="Base URL of the Crawl4AI Docker server (default: http://localhost:11234)"
-    )
-    args = parser.parse_args()
-    
-    dashboard = MonitoringDashboard(base_url=args.url)
-    try:
-        await dashboard.run_dashboard()
-    finally:
-        await dashboard.close()
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

tests/docker/extended_features/demo_proxy_rotation.py

@@ -1,772 +0,0 @@
-#!/usr/bin/env python3
-"""
-Proxy Rotation Demo Script
-
-This script demonstrates real-world usage scenarios for the proxy rotation feature.
-It simulates actual user workflows and shows how to integrate proxy rotation
-into your crawling tasks.
-
-Usage:
-    python demo_proxy_rotation.py
-
-Note: Update the proxy configuration with your actual proxy servers for real testing.
-"""
-
-import asyncio
-import json
-import time
-from datetime import datetime
-from typing import Any, Dict, List
-
-import requests
-from rich import print as rprint
-from rich.console import Console
-
-# Initialize rich console for colored output
-console = Console()
-
-# Configuration
-API_BASE_URL = "http://localhost:11235"
-
-# Import real proxy configuration
-try:
-    from real_proxy_config import (
-        PROXY_POOL_LARGE,
-        PROXY_POOL_MEDIUM,
-        PROXY_POOL_SMALL,
-        REAL_PROXIES,
-    )
-
-    USE_REAL_PROXIES = True
-    console.print(
-        f"[green]✅ Loaded {len(REAL_PROXIES)} real proxies from configuration[/green]"
-    )
-except ImportError:
-    # Fallback to demo proxies if real_proxy_config.py not found
-    REAL_PROXIES = [
-        {
-            "server": "http://proxy1.example.com:8080",
-            "username": "user1",
-            "password": "pass1",
-        },
-        {
-            "server": "http://proxy2.example.com:8080",
-            "username": "user2",
-            "password": "pass2",
-        },
-        {
-            "server": "http://proxy3.example.com:8080",
-            "username": "user3",
-            "password": "pass3",
-        },
-    ]
-    PROXY_POOL_SMALL = REAL_PROXIES[:2]
-    PROXY_POOL_MEDIUM = REAL_PROXIES[:2]
-    PROXY_POOL_LARGE = REAL_PROXIES
-    USE_REAL_PROXIES = False
-    console.print(
-        f"[yellow]⚠️  Using demo proxies (real_proxy_config.py not found)[/yellow]"
-    )
-
-# Alias for backward compatibility
-DEMO_PROXIES = REAL_PROXIES
-
-# Set to True to test with actual proxies, False for demo mode (no proxies, just shows API)
-USE_REAL_PROXIES = False
-
-# Test URLs that help verify proxy rotation
-TEST_URLS = [
-    "https://httpbin.org/ip",  # Shows origin IP
-    "https://httpbin.org/headers",  # Shows all headers
-    "https://httpbin.org/user-agent",  # Shows user agent
-]
-
-
-def print_header(text: str):
-    """Print a formatted header"""
-    console.print(f"\n[cyan]{'=' * 60}[/cyan]")
-    console.print(f"[cyan]{text.center(60)}[/cyan]")
-    console.print(f"[cyan]{'=' * 60}[/cyan]\n")
-
-
-def print_success(text: str):
-    """Print success message"""
-    console.print(f"[green]✅ {text}[/green]")
-
-
-def print_info(text: str):
-    """Print info message"""
-    console.print(f"[blue]ℹ️  {text}[/blue]")
-
-
-def print_warning(text: str):
-    """Print warning message"""
-    console.print(f"[yellow]⚠️  {text}[/yellow]")
-
-
-def print_error(text: str):
-    """Print error message"""
-    console.print(f"[red]❌ {text}[/red]")
-
-
-def check_server_health() -> bool:
-    """Check if the Crawl4AI server is running"""
-    try:
-        response = requests.get(f"{API_BASE_URL}/health", timeout=5)
-        if response.status_code == 200:
-            print_success("Crawl4AI server is running")
-            return True
-        else:
-            print_error(f"Server returned status code: {response.status_code}")
-            return False
-    except Exception as e:
-        print_error(f"Cannot connect to server: {e}")
-        print_warning("Make sure the Crawl4AI server is running on localhost:11235")
-        return False
-
-
-def demo_1_basic_round_robin():
-    """Demo 1: Basic proxy rotation with round robin strategy"""
-    print_header("Demo 1: Basic Round Robin Rotation")
-
-    print_info("Use case: Even distribution across proxies for general crawling")
-    print_info("Strategy: Round Robin - cycles through proxies sequentially\n")
-
-    if USE_REAL_PROXIES:
-        payload = {
-            "urls": [TEST_URLS[0]],  # Just checking IP
-            "proxy_rotation_strategy": "round_robin",
-            "proxies": PROXY_POOL_SMALL,  # Use small pool (3 proxies)
-            "headless": True,
-            "browser_config": {
-                "type": "BrowserConfig",
-                "params": {"headless": True, "verbose": False},
-            },
-            "crawler_config": {
-                "type": "CrawlerRunConfig",
-                "params": {"cache_mode": "bypass", "verbose": False},
-            },
-        }
-    else:
-        print_warning(
-            "Demo mode: Showing API structure without actual proxy connections"
-        )
-        payload = {
-            "urls": [TEST_URLS[0]],
-            "headless": True,
-            "browser_config": {
-                "type": "BrowserConfig",
-                "params": {"headless": True, "verbose": False},
-            },
-            "crawler_config": {
-                "type": "CrawlerRunConfig",
-                "params": {"cache_mode": "bypass", "verbose": False},
-            },
-        }
-
-    console.print(f"[yellow]Request payload:[/yellow]")
-    print(json.dumps(payload, indent=2))
-
-    if USE_REAL_PROXIES:
-        print()
-        print_info("With real proxies, the request would:")
-        print_info("  1. Initialize RoundRobinProxyStrategy")
-        print_info("  2. Cycle through proxy1 → proxy2 → proxy1...")
-        print_info("  3. Each request uses the next proxy in sequence")
-
-    try:
-        start_time = time.time()
-        response = requests.post(f"{API_BASE_URL}/crawl", json=payload, timeout=30)
-        elapsed = time.time() - start_time
-
-        if response.status_code == 200:
-            data = response.json()
-            print_success(f"Request completed in {elapsed:.2f} seconds")
-            print_info(f"Results: {len(data.get('results', []))} URL(s) crawled")
-
-            # Show first result summary
-            if data.get("results"):
-                result = data["results"][0]
-                print_info(f"Success: {result.get('success')}")
-                print_info(f"URL: {result.get('url')}")
-
-            if not USE_REAL_PROXIES:
-                print()
-                print_success(
-                    "✨ API integration works! Add real proxies to test rotation."
-                )
-        else:
-            print_error(f"Request failed: {response.status_code}")
-            if "PROXY_CONNECTION_FAILED" in response.text:
-                print_warning(
-                    "Proxy connection failed - this is expected with example proxies"
-                )
-                print_info(
-                    "Update DEMO_PROXIES and set USE_REAL_PROXIES = True to test with real proxies"
-                )
-            else:
-                print(response.text)
-
-    except Exception as e:
-        print_error(f"Error: {e}")
-
-
-def demo_2_random_stealth():
-    """Demo 2: Random proxy rotation with stealth mode"""
-    print_header("Demo 2: Random Rotation + Stealth Mode")
-
-    print_info("Use case: Unpredictable traffic pattern with anti-bot evasion")
-    print_info("Strategy: Random - unpredictable proxy selection")
-    print_info("Feature: Combined with stealth anti-bot strategy\n")
-
-    payload = {
-        "urls": [TEST_URLS[1]],  # Check headers
-        "proxy_rotation_strategy": "random",
-        "anti_bot_strategy": "stealth",  # Combined with anti-bot
-        "proxies": PROXY_POOL_MEDIUM,  # Use medium pool (5 proxies)
-        "headless": True,
-        "browser_config": {
-            "type": "BrowserConfig",
-            "params": {"headless": True, "enable_stealth": True, "verbose": False},
-        },
-        "crawler_config": {
-            "type": "CrawlerRunConfig",
-            "params": {"cache_mode": "bypass"},
-        },
-    }
-
-    console.print(f"[yellow]Request payload (key parts):[/yellow]")
-    print(
-        json.dumps(
-            {
-                "urls": payload["urls"],
-                "proxy_rotation_strategy": payload["proxy_rotation_strategy"],
-                "anti_bot_strategy": payload["anti_bot_strategy"],
-                "proxies": f"{len(payload['proxies'])} proxies configured",
-            },
-            indent=2,
-        )
-    )
-
-    try:
-        start_time = time.time()
-        response = requests.post(f"{API_BASE_URL}/crawl", json=payload, timeout=30)
-        elapsed = time.time() - start_time
-
-        if response.status_code == 200:
-            data = response.json()
-            print_success(f"Request completed in {elapsed:.2f} seconds")
-            print_success("Random proxy + stealth mode working together!")
-        else:
-            print_error(f"Request failed: {response.status_code}")
-
-    except Exception as e:
-        print_error(f"Error: {e}")
-
-
-def demo_3_least_used_multiple_urls():
-    """Demo 3: Least used strategy with multiple URLs"""
-    print_header("Demo 3: Least Used Strategy (Load Balancing)")
-
-    print_info("Use case: Optimal load distribution across multiple requests")
-    print_info("Strategy: Least Used - balances load across proxy pool")
-    print_info("Feature: Crawling multiple URLs efficiently\n")
-
-    payload = {
-        "urls": TEST_URLS,  # All test URLs
-        "proxy_rotation_strategy": "least_used",
-        "proxies": PROXY_POOL_LARGE,  # Use full pool (all proxies)
-        "headless": True,
-        "browser_config": {
-            "type": "BrowserConfig",
-            "params": {"headless": True, "verbose": False},
-        },
-        "crawler_config": {
-            "type": "CrawlerRunConfig",
-            "params": {
-                "cache_mode": "bypass",
-                "wait_for_images": False,  # Speed up crawling
-                "verbose": False,
-            },
-        },
-    }
-
-    console.print(
-        f"[yellow]Crawling {len(payload['urls'])} URLs with load balancing:[/yellow]"
-    )
-    for i, url in enumerate(payload["urls"], 1):
-        print(f"  {i}. {url}")
-
-    try:
-        start_time = time.time()
-        response = requests.post(f"{API_BASE_URL}/crawl", json=payload, timeout=60)
-        elapsed = time.time() - start_time
-
-        if response.status_code == 200:
-            data = response.json()
-            results = data.get("results", [])
-            print_success(f"Completed {len(results)} URLs in {elapsed:.2f} seconds")
-            print_info(f"Average time per URL: {elapsed / len(results):.2f}s")
-
-            # Show success rate
-            successful = sum(1 for r in results if r.get("success"))
-            print_info(
-                f"Success rate: {successful}/{len(results)} ({successful / len(results) * 100:.1f}%)"
-            )
-        else:
-            print_error(f"Request failed: {response.status_code}")
-
-    except Exception as e:
-        print_error(f"Error: {e}")
-
-
-def demo_4_failure_aware_production():
-    """Demo 4: Failure-aware strategy for production use"""
-    print_header("Demo 4: Failure-Aware Strategy (Production)")
-
-    print_info("Use case: High-availability crawling with automatic recovery")
-    print_info("Strategy: Failure Aware - tracks proxy health")
-    print_info("Feature: Auto-recovery after failures\n")
-
-    payload = {
-        "urls": [TEST_URLS[0]],
-        "proxy_rotation_strategy": "failure_aware",
-        "proxy_failure_threshold": 2,  # Mark unhealthy after 2 failures
-        "proxy_recovery_time": 120,  # 2 minutes recovery time
-        "proxies": PROXY_POOL_MEDIUM,  # Use medium pool (5 proxies)
-        "headless": True,
-        "browser_config": {
-            "type": "BrowserConfig",
-            "params": {"headless": True, "verbose": False},
-        },
-        "crawler_config": {
-            "type": "CrawlerRunConfig",
-            "params": {"cache_mode": "bypass"},
-        },
-    }
-
-    console.print(f"[yellow]Configuration:[/yellow]")
-    print(f"  Failure threshold: {payload['proxy_failure_threshold']} failures")
-    print(f"  Recovery time: {payload['proxy_recovery_time']} seconds")
-    print(f"  Proxy pool size: {len(payload['proxies'])} proxies")
-
-    try:
-        start_time = time.time()
-        response = requests.post(f"{API_BASE_URL}/crawl", json=payload, timeout=30)
-        elapsed = time.time() - start_time
-
-        if response.status_code == 200:
-            data = response.json()
-            print_success(f"Request completed in {elapsed:.2f} seconds")
-            print_success("Failure-aware strategy initialized successfully")
-            print_info("The strategy will now track proxy health automatically")
-        else:
-            print_error(f"Request failed: {response.status_code}")
-
-    except Exception as e:
-        print_error(f"Error: {e}")
-
-
-def demo_5_streaming_with_proxies():
-    """Demo 5: Streaming endpoint with proxy rotation"""
-    print_header("Demo 5: Streaming with Proxy Rotation")
-
-    print_info("Use case: Real-time results with proxy rotation")
-    print_info("Strategy: Random - varies proxies across stream")
-    print_info("Feature: Streaming endpoint support\n")
-
-    payload = {
-        "urls": TEST_URLS[:2],  # First 2 URLs
-        "proxy_rotation_strategy": "random",
-        "proxies": PROXY_POOL_SMALL,  # Use small pool (3 proxies)
-        "headless": True,
-        "browser_config": {
-            "type": "BrowserConfig",
-            "params": {"headless": True, "verbose": False},
-        },
-        "crawler_config": {
-            "type": "CrawlerRunConfig",
-            "params": {"stream": True, "cache_mode": "bypass", "verbose": False},
-        },
-    }
-
-    print_info("Streaming 2 URLs with random proxy rotation...")
-
-    try:
-        start_time = time.time()
-        response = requests.post(
-            f"{API_BASE_URL}/crawl/stream", json=payload, timeout=60, stream=True
-        )
-
-        if response.status_code == 200:
-            results_count = 0
-            for line in response.iter_lines():
-                if line:
-                    try:
-                        data = json.loads(line.decode("utf-8"))
-                        if data.get("status") == "processing":
-                            print_info(f"Processing: {data.get('url', 'unknown')}")
-                        elif data.get("status") == "completed":
-                            results_count += 1
-                            print_success(f"Completed: {data.get('url', 'unknown')}")
-                    except json.JSONDecodeError:
-                        pass
-
-            elapsed = time.time() - start_time
-            print_success(
-                f"\nStreaming completed: {results_count} results in {elapsed:.2f}s"
-            )
-        else:
-            print_error(f"Streaming failed: {response.status_code}")
-
-    except Exception as e:
-        print_error(f"Error: {e}")
-
-
-def demo_6_error_handling():
-    """Demo 6: Error handling demonstration"""
-    print_header("Demo 6: Error Handling")
-
-    print_info("Demonstrating how the system handles errors gracefully\n")
-
-    # Test 1: Invalid strategy
-    console.print(f"[yellow]Test 1: Invalid strategy name[/yellow]")
-    payload = {
-        "urls": [TEST_URLS[0]],
-        "proxy_rotation_strategy": "invalid_strategy",
-        "proxies": [PROXY_POOL_SMALL[0]],  # Use just 1 proxy
-        "headless": True,
-    }
-
-    try:
-        response = requests.post(f"{API_BASE_URL}/crawl", json=payload, timeout=10)
-        if response.status_code != 200:
-            print_error(
-                f"Expected error: {response.json().get('detail', 'Unknown error')}"
-            )
-        else:
-            print_warning("Unexpected: Request succeeded")
-    except Exception as e:
-        print_error(f"Error: {e}")
-
-    print()
-
-    # Test 2: Missing server field
-    console.print(f"[yellow]Test 2: Invalid proxy configuration[/yellow]")
-    payload = {
-        "urls": [TEST_URLS[0]],
-        "proxy_rotation_strategy": "round_robin",
-        "proxies": [{"username": "user1"}],  # Missing server
-        "headless": True,
-    }
-
-    try:
-        response = requests.post(f"{API_BASE_URL}/crawl", json=payload, timeout=10)
-        if response.status_code != 200:
-            print_error(
-                f"Expected error: {response.json().get('detail', 'Unknown error')}"
-            )
-        else:
-            print_warning("Unexpected: Request succeeded")
-    except Exception as e:
-        print_error(f"Error: {e}")
-
-    print()
-    print_success("Error handling working as expected!")
-
-
-def demo_7_real_world_scenario():
-    """Demo 7: Real-world e-commerce price monitoring scenario"""
-    print_header("Demo 7: Real-World Scenario - Price Monitoring")
-
-    print_info("Scenario: Monitoring multiple product pages with high availability")
-    print_info("Requirements: Anti-detection + Proxy rotation + Fault tolerance\n")
-
-    # Simulated product URLs (using httpbin for demo)
-    product_urls = [
-        "https://httpbin.org/delay/1",  # Simulates slow page
-        "https://httpbin.org/html",  # Simulates product page
-        "https://httpbin.org/json",  # Simulates API endpoint
-    ]
-
-    payload = {
-        "urls": product_urls,
-        "anti_bot_strategy": "stealth",
-        "proxy_rotation_strategy": "failure_aware",
-        "proxy_failure_threshold": 2,
-        "proxy_recovery_time": 180,
-        "proxies": PROXY_POOL_LARGE,  # Use full pool for high availability
-        "headless": True,
-        "browser_config": {
-            "type": "BrowserConfig",
-            "params": {"headless": True, "enable_stealth": True, "verbose": False},
-        },
-        "crawler_config": {
-            "type": "CrawlerRunConfig",
-            "params": {
-                "cache_mode": "bypass",
-                "page_timeout": 30000,
-                "wait_for_images": False,
-                "verbose": False,
-            },
-        },
-    }
-
-    console.print(f"[yellow]Configuration:[/yellow]")
-    print(f"  URLs to monitor: {len(product_urls)}")
-    print(f"  Anti-bot strategy: stealth")
-    print(f"  Proxy strategy: failure_aware")
-    print(f"  Proxy pool: {len(DEMO_PROXIES)} proxies")
-    print()
-
-    print_info("Starting price monitoring crawl...")
-
-    try:
-        start_time = time.time()
-        response = requests.post(f"{API_BASE_URL}/crawl", json=payload, timeout=90)
-        elapsed = time.time() - start_time
-
-        if response.status_code == 200:
-            data = response.json()
-            results = data.get("results", [])
-
-            print_success(f"Monitoring completed in {elapsed:.2f} seconds\n")
-
-            # Detailed results
-            console.print(f"[yellow]Results Summary:[/yellow]")
-            for i, result in enumerate(results, 1):
-                url = result.get("url", "unknown")
-                success = result.get("success", False)
-                status = "✅ Success" if success else "❌ Failed"
-                print(f"  {i}. {status} - {url}")
-
-            successful = sum(1 for r in results if r.get("success"))
-            print()
-            print_info(
-                f"Success rate: {successful}/{len(results)} ({successful / len(results) * 100:.1f}%)"
-            )
-            print_info(f"Average time per product: {elapsed / len(results):.2f}s")
-
-            print()
-            print_success("✨ Real-world scenario completed successfully!")
-            print_info("This configuration is production-ready for:")
-            print_info("  - E-commerce price monitoring")
-            print_info("  - Competitive analysis")
-            print_info("  - Market research")
-            print_info("  - Any high-availability crawling needs")
-        else:
-            print_error(f"Request failed: {response.status_code}")
-            print(response.text)
-
-    except Exception as e:
-        print_error(f"Error: {e}")
-
-
-def show_python_integration_example():
-    """Show Python integration code example"""
-    print_header("Python Integration Example")
-
-    code = '''
-import requests
-import json
-
-class ProxyCrawler:
-    """Example class for integrating proxy rotation into your application"""
-    
-    def __init__(self, api_url="http://localhost:11235"):
-        self.api_url = api_url
-        self.proxies = [
-            {"server": "http://proxy1.com:8080", "username": "user", "password": "pass"},
-            {"server": "http://proxy2.com:8080", "username": "user", "password": "pass"},
-        ]
-    
-    def crawl_with_proxies(self, urls, strategy="round_robin"):
-        """Crawl URLs with proxy rotation"""
-        payload = {
-            "urls": urls,
-            "proxy_rotation_strategy": strategy,
-            "proxies": self.proxies,
-            "headless": True,
-            "crawler_config": {
-                "type": "CrawlerRunConfig",
-                "params": {"cache_mode": "bypass"}
-            }
-        }
-        
-        response = requests.post(f"{self.api_url}/crawl", json=payload, timeout=60)
-        return response.json()
-    
-    def monitor_prices(self, product_urls):
-        """Monitor product prices with high availability"""
-        payload = {
-            "urls": product_urls,
-            "anti_bot_strategy": "stealth",
-            "proxy_rotation_strategy": "failure_aware",
-            "proxy_failure_threshold": 2,
-            "proxies": self.proxies,
-            "headless": True
-        }
-        
-        response = requests.post(f"{self.api_url}/crawl", json=payload, timeout=120)
-        return response.json()
-
-# Usage
-crawler = ProxyCrawler()
-
-# Simple crawling
-results = crawler.crawl_with_proxies(
-    urls=["https://example.com"],
-    strategy="round_robin"
-)
-
-# Price monitoring
-product_results = crawler.monitor_prices(
-    product_urls=["https://shop.example.com/product1", "https://shop.example.com/product2"]
-)
-'''
-
-    console.print(f"[green]{code}[/green]")
-    print_info("Copy this code to integrate proxy rotation into your application!")
-
-
-def demo_0_proxy_setup_guide():
-    """Demo 0: Guide for setting up real proxies"""
-    print_header("Proxy Setup Guide")
-
-    print_info("This demo can run in two modes:\n")
-
-    console.print(f"[yellow]1. DEMO MODE (Current):[/yellow]")
-    print("   - Tests API integration without proxies")
-    print("   - Shows request/response structure")
-    print("   - Safe to run without proxy servers\n")
-
-    console.print(f"[yellow]2. REAL PROXY MODE:[/yellow]")
-    print("   - Tests actual proxy rotation")
-    print("   - Requires valid proxy servers")
-    print("   - Shows real proxy switching in action\n")
-
-    console.print(f"[green]To enable real proxy testing:[/green]")
-    print("   1. Update DEMO_PROXIES with your actual proxy servers:")
-    print()
-    console.print("[cyan]      DEMO_PROXIES = [")
-    console.print(
-        "          {'server': 'http://your-proxy1.com:8080', 'username': 'user', 'password': 'pass'},"
-    )
-    console.print(
-        "          {'server': 'http://your-proxy2.com:8080', 'username': 'user', 'password': 'pass'},"
-    )
-    console.print("      ][/cyan]")
-    print()
-    console.print(f"   2. Set: [cyan]USE_REAL_PROXIES = True[/cyan]")
-    print()
-
-    console.print(f"[yellow]Popular Proxy Providers:[/yellow]")
-    print("   - Bright Data (formerly Luminati)")
-    print("   - Oxylabs")
-    print("   - Smartproxy")
-    print("   - ProxyMesh")
-    print("   - Your own proxy servers")
-    print()
-
-    if USE_REAL_PROXIES:
-        print_success("Real proxy mode is ENABLED")
-        print_info(f"Using {len(DEMO_PROXIES)} configured proxies")
-    else:
-        print_info("Demo mode is active (USE_REAL_PROXIES = False)")
-        print_info(
-            "API structure will be demonstrated without actual proxy connections"
-        )
-
-
-def main():
-    """Main demo runner"""
-    console.print(f"""
-[cyan]╔══════════════════════════════════════════════════════════╗
-║                                                          ║
-║          Crawl4AI Proxy Rotation Demo Suite             ║
-║                                                          ║
-║  Demonstrating real-world proxy rotation scenarios      ║
-║                                                          ║
-╚══════════════════════════════════════════════════════════╝[/cyan]
-""")
-
-    if USE_REAL_PROXIES:
-        print_success(f"✨ Using {len(REAL_PROXIES)} real Webshare proxies")
-        print_info(f"📊 Proxy pools configured:")
-        print_info(f"   • Small pool: {len(PROXY_POOL_SMALL)} proxies (quick tests)")
-        print_info(f"   • Medium pool: {len(PROXY_POOL_MEDIUM)} proxies (balanced)")
-        print_info(
-            f"   • Large pool: {len(PROXY_POOL_LARGE)} proxies (high availability)"
-        )
-    else:
-        print_warning("⚠️  Using demo proxy configuration (won't connect)")
-        print_info("To use real proxies, create real_proxy_config.py with your proxies")
-    print()
-
-    # Check server health
-    if not check_server_health():
-        print()
-        print_error("Please start the Crawl4AI server first:")
-        print_info("cd deploy/docker && docker-compose up")
-        print_info("or run: ./dev.sh")
-        return
-
-    print()
-    input(f"[yellow]Press Enter to start the demos...[/yellow]")
-
-    # Run all demos
-    demos = [
-        demo_0_proxy_setup_guide,
-        demo_1_basic_round_robin,
-        demo_2_random_stealth,
-        demo_3_least_used_multiple_urls,
-        demo_4_failure_aware_production,
-        demo_5_streaming_with_proxies,
-        demo_6_error_handling,
-        demo_7_real_world_scenario,
-    ]
-
-    for i, demo in enumerate(demos, 1):
-        try:
-            demo()
-            if i < len(demos):
-                print()
-                input(f"[yellow]Press Enter to continue to next demo...[/yellow]")
-        except KeyboardInterrupt:
-            print()
-            print_warning("Demo interrupted by user")
-            break
-        except Exception as e:
-            print_error(f"Demo failed: {e}")
-            import traceback
-
-            traceback.print_exc()
-
-    # Show integration example
-    print()
-    show_python_integration_example()
-
-    # Summary
-    print_header("Demo Suite Complete!")
-    print_success("You've seen all major proxy rotation features!")
-    print()
-    print_info("Next steps:")
-    print_info("  1. Update DEMO_PROXIES with your actual proxy servers")
-    print_info("  2. Run: python test_proxy_rotation_strategies.py (full test suite)")
-    print_info("  3. Read: PROXY_ROTATION_STRATEGY_DOCS.md (complete documentation)")
-    print_info("  4. Integrate into your application using the examples above")
-    print()
-    console.print(f"[cyan]Happy crawling! 🚀[/cyan]")
-
-
-if __name__ == "__main__":
-    try:
-        main()
-    except KeyboardInterrupt:
-        print()
-        print_warning("\nDemo interrupted. Goodbye!")
-    except Exception as e:
-        print_error(f"\nUnexpected error: {e}")
-        import traceback
-
-        traceback.print_exc()

tests/docker/extended_features/demo_seed_endpoint.py

@@ -1,300 +0,0 @@
-#!/usr/bin/env python3
-"""
-Demo: How users will call the Seed endpoint
-This shows practical examples of how developers would use the seed endpoint
-in their applications to discover URLs for crawling.
-"""
-
-import asyncio
-from typing import Any, Dict
-
-import aiohttp
-
-# Configuration
-API_BASE_URL = "http://localhost:11235"
-API_TOKEN = None  # Set if your API requires authentication
-
-
-class SeedEndpointDemo:
-    def __init__(self, base_url: str = API_BASE_URL, token: str = None):
-        self.base_url = base_url
-        self.headers = {"Content-Type": "application/json"}
-        if token:
-            self.headers["Authorization"] = f"Bearer {token}"
-
-    async def call_seed_endpoint(
-        self, url: str, max_urls: int = 20, filter_type: str = "all", **kwargs
-    ) -> Dict[str, Any]:
-        """Make a call to the seed endpoint"""
-        # The seed endpoint expects 'url' and config with other parameters
-        config = {
-            "max_urls": max_urls,
-            "filter_type": filter_type,
-            **kwargs,
-        }
-        payload = {
-            "url": url,
-            "config": config,
-        }
-
-        async with aiohttp.ClientSession() as session:
-            async with session.post(
-                f"{self.base_url}/seed", headers=self.headers, json=payload
-            ) as response:
-                if response.status == 200:
-                    result = await response.json()
-                    # Extract the nested seeded_urls from the response
-                    seed_data = result.get('seed_url', {})
-                    if isinstance(seed_data, dict):
-                        return seed_data
-                    else:
-                        return {'seeded_urls': seed_data or [], 'count': len(seed_data or [])}
-                else:
-                    error_text = await response.text()
-                    raise Exception(f"API Error {response.status}: {error_text}")
-
-    async def demo_news_site_seeding(self):
-        """Demo: Seed URLs from a news website"""
-        print("🗞️  Demo: Seeding URLs from a News Website")
-        print("=" * 50)
-
-        try:
-            result = await self.call_seed_endpoint(
-                url="https://techcrunch.com",
-                max_urls=15,
-                source="sitemap",  # Try sitemap first
-                live_check=True,
-            )
-
-            urls_found = len(result.get('seeded_urls', []))
-            print(f"✅ Found {urls_found} URLs")
-            
-            if 'message' in result:
-                print(f"ℹ️  Server message: {result['message']}")
-                
-            processing_time = result.get('processing_time', 'N/A')
-            print(f"📊 Seed completed in: {processing_time} seconds")
-
-            # Show first 5 URLs as example
-            seeded_urls = result.get("seeded_urls", [])
-            for i, url in enumerate(seeded_urls[:5]):
-                print(f"  {i + 1}. {url}")
-
-            if len(seeded_urls) > 5:
-                print(f"  ... and {len(seeded_urls) - 5} more URLs")
-            elif len(seeded_urls) == 0:
-                print("  💡 Note: No URLs found. This could be because:")
-                print("     - The website doesn't have an accessible sitemap")
-                print("     - The seeding configuration needs adjustment")
-                print("     - Try different source options like 'cc' (Common Crawl)")
-
-        except Exception as e:
-            print(f"❌ Error: {e}")
-            print("  💡 This might be a connectivity issue or server problem")
-
-    async def demo_ecommerce_seeding(self):
-        """Demo: Seed product URLs from an e-commerce site"""
-        print("\n🛒 Demo: Seeding Product URLs from E-commerce")
-        print("=" * 50)
-        print("💡 Note: This demonstrates configuration for e-commerce sites")
-
-        try:
-            result = await self.call_seed_endpoint(
-                url="https://example-shop.com",
-                max_urls=25,
-                source="sitemap+cc",
-                pattern="*/product/*",  # Focus on product pages
-                live_check=False,
-            )
-
-            urls_found = len(result.get('seeded_urls', []))
-            print(f"✅ Found {urls_found} product URLs")
-            
-            if 'message' in result:
-                print(f"ℹ️  Server message: {result['message']}")
-
-            # Show examples if any found
-            seeded_urls = result.get("seeded_urls", [])
-            if seeded_urls:
-                print("📦 Product URLs discovered:")
-                for i, url in enumerate(seeded_urls[:3]):
-                    print(f"  {i + 1}. {url}")
-            else:
-                print("💡 For real e-commerce seeding, you would:")
-                print("   • Use actual e-commerce site URLs")
-                print("   • Set patterns like '*/product/*' or '*/item/*'")
-                print("   • Enable live_check to verify product page availability")
-                print("   • Use appropriate max_urls based on catalog size")
-
-        except Exception as e:
-            print(f"❌ Error: {e}")
-            print("   This is expected for the example URL")
-
-    async def demo_documentation_seeding(self):
-        """Demo: Seed documentation pages"""
-        print("\n📚 Demo: Seeding Documentation Pages")
-        print("=" * 50)
-
-        try:
-            result = await self.call_seed_endpoint(
-                url="https://docs.python.org",
-                max_urls=30,
-                source="sitemap",
-                pattern="*/library/*",  # Focus on library documentation
-                live_check=False,
-            )
-
-            urls_found = len(result.get('seeded_urls', []))
-            print(f"✅ Found {urls_found} documentation URLs")
-            
-            if 'message' in result:
-                print(f"ℹ️  Server message: {result['message']}")
-
-            # Analyze URL structure if URLs found
-            seeded_urls = result.get("seeded_urls", [])
-            if seeded_urls:
-                sections = {"library": 0, "tutorial": 0, "reference": 0, "other": 0}
-
-                for url in seeded_urls:
-                    if "/library/" in url:
-                        sections["library"] += 1
-                    elif "/tutorial/" in url:
-                        sections["tutorial"] += 1
-                    elif "/reference/" in url:
-                        sections["reference"] += 1
-                    else:
-                        sections["other"] += 1
-
-                print("📊 URL distribution:")
-                for section, count in sections.items():
-                    if count > 0:
-                        print(f"  {section.title()}: {count} URLs")
-                        
-                # Show examples
-                print("\n📖 Example URLs:")
-                for i, url in enumerate(seeded_urls[:3]):
-                    print(f"  {i + 1}. {url}")
-            else:
-                print("💡 For documentation seeding, you would typically:")
-                print("   • Use sites with comprehensive sitemaps like docs.python.org")
-                print("   • Set patterns to focus on specific sections ('/library/', '/tutorial/')")
-                print("   • Consider using 'cc' source for broader coverage")
-
-        except Exception as e:
-            print(f"❌ Error: {e}")
-
-    async def demo_seeding_sources(self):
-        """Demo: Different seeding sources available"""
-        print("\n<EFBFBD> Demo: Understanding Seeding Sources")
-        print("=" * 50)
-        
-        print("📖 Available seeding sources:")
-        print("  • 'sitemap': Discovers URLs from website's sitemap.xml")
-        print("  • 'cc': Uses Common Crawl database for URL discovery")
-        print("  • 'sitemap+cc': Combines both sources (default)")
-        print()
-        
-        test_url = "https://docs.python.org"
-        sources = ["sitemap", "cc", "sitemap+cc"]
-        
-        for source in sources:
-            print(f"🧪 Testing source: '{source}'")
-            try:
-                result = await self.call_seed_endpoint(
-                    url=test_url,
-                    max_urls=5,
-                    source=source,
-                    live_check=False,  # Faster for demo
-                )
-                
-                urls_found = len(result.get('seeded_urls', []))
-                print(f"  ✅ {source}: Found {urls_found} URLs")
-                
-                if urls_found > 0:
-                    # Show first URL as example
-                    first_url = result.get('seeded_urls', [])[0]
-                    print(f"     Example: {first_url}")
-                elif 'message' in result:
-                    print(f"     Info: {result['message']}")
-                    
-            except Exception as e:
-                print(f"  ❌ {source}: Error - {e}")
-            
-            print()  # Space between tests
-
-    async def demo_working_example(self):
-        """Demo: A realistic working example"""
-        print("\n✨ Demo: Working Example with Live Seeding")
-        print("=" * 50)
-        
-        print("🎯 Testing with a site that likely has good sitemap support...")
-        
-        try:
-            # Use a site that's more likely to have a working sitemap
-            result = await self.call_seed_endpoint(
-                url="https://github.com",
-                max_urls=10,
-                source="sitemap",
-                pattern="*/blog/*",  # Focus on blog posts
-                live_check=False,
-            )
-            
-            urls_found = len(result.get('seeded_urls', []))
-            print(f"✅ Found {urls_found} URLs from GitHub")
-            
-            if urls_found > 0:
-                print("🎉 Success! Here are some discovered URLs:")
-                for i, url in enumerate(result.get('seeded_urls', [])[:3]):
-                    print(f"  {i + 1}. {url}")
-                print()
-                print("💡 This demonstrates that seeding works when:")
-                print("   • The target site has an accessible sitemap")
-                print("   • The configuration matches available content")
-                print("   • Network connectivity allows sitemap access")
-            else:
-                print("ℹ️  No URLs found, but this is normal for demo purposes.")
-                print("💡 In real usage, you would:")
-                print("   • Test with sites you know have sitemaps")
-                print("   • Use appropriate URL patterns for your use case")
-                print("   • Consider using 'cc' source for broader discovery")
-                
-        except Exception as e:
-            print(f"❌ Error: {e}")
-            print("💡 This might indicate:")
-            print("   • Network connectivity issues")
-            print("   • Server configuration problems")
-            print("   • Need to adjust seeding parameters")
-
-
-async def main():
-    """Run all seed endpoint demos"""
-    print("🌱 Crawl4AI Seed Endpoint - User Demo")
-    print("=" * 60)
-    print("This demo shows how developers use the seed endpoint")
-    print("to discover URLs for their crawling workflows.\n")
-
-    demo = SeedEndpointDemo()
-
-    # Run individual demos
-    await demo.demo_news_site_seeding()
-    await demo.demo_ecommerce_seeding()
-    await demo.demo_documentation_seeding()
-    await demo.demo_seeding_sources()
-    await demo.demo_working_example()
-
-    print("\n🎉 Demo completed!")
-    print("\n📚 Key Takeaways:")
-    print("1. Seed endpoint discovers URLs from sitemaps and Common Crawl")
-    print("2. Different sources ('sitemap', 'cc', 'sitemap+cc') offer different coverage")
-    print("3. URL patterns help filter discovered content to your needs")
-    print("4. Live checking verifies URL accessibility but slows discovery")
-    print("5. Success depends on target site's sitemap availability")
-    print("\n💡 Next steps for your application:")
-    print("1. Test with your target websites to verify sitemap availability")
-    print("2. Choose appropriate seeding sources for your use case")
-    print("3. Use discovered URLs as input for your crawling pipeline")
-    print("4. Consider fallback strategies if seeding returns few results")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

tests/docker/extended_features/quick_proxy_test.py

@@ -1,304 +0,0 @@
-#!/usr/bin/env python3
-"""
-Quick Proxy Rotation Test
-
-A simple script to quickly verify the proxy rotation feature is working.
-This tests the API integration and strategy initialization without requiring
-actual proxy servers.
-
-Usage:
-    python quick_proxy_test.py
-"""
-
-import requests
-from rich.console import Console
-
-console = Console()
-
-API_URL = "http://localhost:11235"
-
-
-def test_api_accepts_proxy_params():
-    """Test 1: Verify API accepts proxy rotation parameters"""
-    console.print(f"\n[cyan]{'=' * 60}[/cyan]")
-    console.print(f"[cyan]Test 1: API Parameter Validation[/cyan]")
-    console.print(f"[cyan]{'=' * 60}[/cyan]\n")
-
-    # Test valid strategy names
-    strategies = ["round_robin", "random", "least_used", "failure_aware"]
-
-    for strategy in strategies:
-        payload = {
-            "urls": ["https://httpbin.org/html"],
-            "proxy_rotation_strategy": strategy,
-            "proxies": [
-                {
-                    "server": "http://proxy1.com:8080",
-                    "username": "user",
-                    "password": "pass",
-                }
-            ],
-            "headless": True,
-        }
-
-        console.print(f"Testing strategy: [yellow]{strategy}[/yellow]")
-
-        try:
-            # We expect this to fail on proxy connection, but API should accept it
-            response = requests.post(f"{API_URL}/crawl", json=payload, timeout=10)
-
-            if response.status_code == 200:
-                console.print(f"  [green]✅ API accepted {strategy} strategy[/green]")
-            elif (
-                response.status_code == 500
-                and "PROXY_CONNECTION_FAILED" in response.text
-            ):
-                console.print(
-                    f"  [green]✅ API accepted {strategy} strategy (proxy connection failed as expected)[/green]"
-                )
-            elif response.status_code == 422:
-                console.print(f"  [red]❌ API rejected {strategy} strategy[/red]")
-                print(f"     {response.json()}")
-            else:
-                console.print(
-                    f"  [yellow]⚠️  Unexpected response: {response.status_code}[/yellow]"
-                )
-
-        except requests.Timeout:
-            console.print(f"  [yellow]⚠️  Request timeout[/yellow]")
-        except Exception as e:
-            console.print(f"  [red]❌ Error: {e}[/red]")
-
-
-def test_invalid_strategy():
-    """Test 2: Verify API rejects invalid strategies"""
-    console.print(f"\n[cyan]{'=' * 60}[/cyan]")
-    console.print(f"[cyan]Test 2: Invalid Strategy Rejection[/cyan]")
-    console.print(f"[cyan]{'=' * 60}[/cyan]\n")
-
-    payload = {
-        "urls": ["https://httpbin.org/html"],
-        "proxy_rotation_strategy": "invalid_strategy",
-        "proxies": [{"server": "http://proxy1.com:8080"}],
-        "headless": True,
-    }
-
-    console.print(f"Testing invalid strategy: [yellow]invalid_strategy[/yellow]")
-
-    try:
-        response = requests.post(f"{API_URL}/crawl", json=payload, timeout=10)
-
-        if response.status_code == 422:
-            console.print(f"[green]✅ API correctly rejected invalid strategy[/green]")
-            error = response.json()
-            if isinstance(error, dict) and "detail" in error:
-                print(f"   Validation message: {error['detail'][0]['msg']}")
-        else:
-            console.print(f"[red]❌ API did not reject invalid strategy[/red]")
-
-    except Exception as e:
-        console.print(f"[red]❌ Error: {e}[/red]")
-
-
-def test_optional_params():
-    """Test 3: Verify failure-aware optional parameters"""
-    console.print(f"\n[cyan]{'=' * 60}[/cyan]")
-    console.print(f"[cyan]Test 3: Optional Parameters[/cyan]")
-    console.print(f"[cyan]{'=' * 60}[/cyan]\n")
-
-    payload = {
-        "urls": ["https://httpbin.org/html"],
-        "proxy_rotation_strategy": "failure_aware",
-        "proxy_failure_threshold": 5,  # Custom threshold
-        "proxy_recovery_time": 600,  # Custom recovery time
-        "proxies": [
-            {"server": "http://proxy1.com:8080", "username": "user", "password": "pass"}
-        ],
-        "headless": True,
-    }
-
-    print(f"Testing failure-aware with custom parameters:")
-    print(f"  - proxy_failure_threshold: {payload['proxy_failure_threshold']}")
-    print(f"  - proxy_recovery_time: {payload['proxy_recovery_time']}")
-
-    try:
-        response = requests.post(f"{API_URL}/crawl", json=payload, timeout=10)
-
-        if response.status_code in [200, 500]:  # 500 is ok (proxy connection fails)
-            console.print(
-                f"[green]✅ API accepted custom failure-aware parameters[/green]"
-            )
-        elif response.status_code == 422:
-            console.print(f"[red]❌ API rejected custom parameters[/red]")
-            print(response.json())
-        else:
-            console.print(
-                f"[yellow]⚠️  Unexpected response: {response.status_code}[/yellow]"
-            )
-
-    except Exception as e:
-        console.print(f"[red]❌ Error: {e}[/red]")
-
-
-def test_without_proxies():
-    """Test 4: Normal crawl without proxy rotation (baseline)"""
-    console.print(f"\n[cyan]{'=' * 60}[/cyan]")
-    console.print(f"[cyan]Test 4: Baseline Crawl (No Proxies)[/cyan]")
-    console.print(f"[cyan]{'=' * 60}[/cyan]\n")
-
-    payload = {
-        "urls": ["https://httpbin.org/html"],
-        "headless": True,
-        "browser_config": {
-            "type": "BrowserConfig",
-            "params": {"headless": True, "verbose": False},
-        },
-        "crawler_config": {
-            "type": "CrawlerRunConfig",
-            "params": {"cache_mode": "bypass", "verbose": False},
-        },
-    }
-
-    print("Testing normal crawl without proxy rotation...")
-
-    try:
-        response = requests.post(f"{API_URL}/crawl", json=payload, timeout=30)
-
-        if response.status_code == 200:
-            data = response.json()
-            results = data.get("results", [])
-            if results and results[0].get("success"):
-                console.print(f"[green]✅ Baseline crawl successful[/green]")
-                print(f"   URL: {results[0].get('url')}")
-                print(f"   Content length: {len(results[0].get('html', ''))} chars")
-            else:
-                console.print(f"[yellow]⚠️  Crawl completed but with issues[/yellow]")
-        else:
-            console.print(
-                f"[red]❌ Baseline crawl failed: {response.status_code}[/red]"
-            )
-
-    except Exception as e:
-        console.print(f"[red]❌ Error: {e}[/red]")
-
-
-def test_proxy_config_formats():
-    """Test 5: Different proxy configuration formats"""
-    console.print(f"\n[cyan]{'=' * 60}[/cyan]")
-    console.print(f"[cyan]Test 5: Proxy Configuration Formats[/cyan]")
-    console.print(f"[cyan]{'=' * 60}[/cyan]\n")
-
-    test_cases = [
-        {
-            "name": "With username/password",
-            "proxy": {
-                "server": "http://proxy.com:8080",
-                "username": "user",
-                "password": "pass",
-            },
-        },
-        {"name": "Server only", "proxy": {"server": "http://proxy.com:8080"}},
-        {
-            "name": "HTTPS proxy",
-            "proxy": {
-                "server": "https://proxy.com:8080",
-                "username": "user",
-                "password": "pass",
-            },
-        },
-    ]
-
-    for test_case in test_cases:
-        console.print(f"Testing: [yellow]{test_case['name']}[/yellow]")
-
-        payload = {
-            "urls": ["https://httpbin.org/html"],
-            "proxy_rotation_strategy": "round_robin",
-            "proxies": [test_case["proxy"]],
-            "headless": True,
-        }
-
-        try:
-            response = requests.post(f"{API_URL}/crawl", json=payload, timeout=10)
-
-            if response.status_code in [200, 500]:
-                console.print(f"  [green]✅ Format accepted[/green]")
-            elif response.status_code == 422:
-                console.print(f"  [red]❌ Format rejected[/red]")
-                print(f"     {response.json()}")
-            else:
-                console.print(
-                    f"  [yellow]⚠️  Unexpected: {response.status_code}[/yellow]"
-                )
-
-        except Exception as e:
-            console.print(f"  [red]❌ Error: {e}[/red]")
-
-
-def main():
-    console.print(f"""
-[cyan]╔══════════════════════════════════════════════════════════╗
-║                                                          ║
-║        Quick Proxy Rotation Feature Test                ║
-║                                                          ║
-║  Verifying API integration without real proxies         ║
-║                                                          ║
-╚══════════════════════════════════════════════════════════╝[/cyan]
-""")
-
-    # Check server
-    try:
-        response = requests.get(f"{API_URL}/health", timeout=5)
-        if response.status_code == 200:
-            console.print(f"[green]✅ Server is running at {API_URL}[/green]\n")
-        else:
-            console.print(
-                f"[red]❌ Server returned status {response.status_code}[/red]\n"
-            )
-            return
-    except Exception as e:
-        console.print(f"[red]❌ Cannot connect to server: {e}[/red]")
-        console.print(
-            f"[yellow]Make sure Crawl4AI server is running on {API_URL}[/yellow]\n"
-        )
-        return
-
-    # Run tests
-    test_api_accepts_proxy_params()
-    test_invalid_strategy()
-    test_optional_params()
-    test_without_proxies()
-    test_proxy_config_formats()
-
-    # Summary
-    console.print(f"\n[cyan]{'=' * 60}[/cyan]")
-    console.print(f"[cyan]Test Summary[/cyan]")
-    console.print(f"[cyan]{'=' * 60}[/cyan]\n")
-
-    console.print(f"[green]✅ Proxy rotation feature is integrated correctly![/green]")
-    print()
-    console.print(f"[yellow]What was tested:[/yellow]")
-    print("  • All 4 rotation strategies accepted by API")
-    print("  • Invalid strategies properly rejected")
-    print("  • Custom failure-aware parameters work")
-    print("  • Different proxy config formats accepted")
-    print("  • Baseline crawling still works")
-    print()
-    console.print(f"[yellow]Next steps:[/yellow]")
-    print("  1. Add real proxy servers to test actual rotation")
-    print("  2. Run: python demo_proxy_rotation.py (full demo)")
-    print("  3. Run: python test_proxy_rotation_strategies.py (comprehensive tests)")
-    print()
-    console.print(f"[cyan]🎉 Feature is ready for production![/cyan]\n")
-
-
-if __name__ == "__main__":
-    try:
-        main()
-    except KeyboardInterrupt:
-        console.print(f"\n[yellow]Test interrupted[/yellow]")
-    except Exception as e:
-        console.print(f"\n[red]Unexpected error: {e}[/red]")
-        import traceback
-
-        traceback.print_exc()

tests/docker/extended_features/test_adapter_chain.py

@@ -1,113 +0,0 @@
-#!/usr/bin/env python3
-"""
-Test what's actually happening with the adapters in the API
-"""
-
-import asyncio
-import os
-import sys
-
-import pytest
-
-# Add the project root to Python path
-sys.path.insert(0, os.getcwd())
-sys.path.insert(0, os.path.join(os.getcwd(), "deploy", "docker"))
-
-
-@pytest.mark.asyncio
-async def test_adapter_chain():
-    """Test the complete adapter chain from API to crawler"""
-    print("🔍 Testing Complete Adapter Chain")
-    print("=" * 50)
-
-    try:
-        # Import the API functions
-        from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
-        from deploy.docker.api import _apply_headless_setting, _get_browser_adapter
-        from deploy.docker.crawler_pool import get_crawler
-
-        print("✅ Successfully imported all functions")
-
-        # Test different strategies
-        strategies = ["default", "stealth", "undetected"]
-
-        for strategy in strategies:
-            print(f"\n🧪 Testing {strategy} strategy:")
-            print("-" * 30)
-
-            try:
-                # Step 1: Create browser config
-                browser_config = BrowserConfig(headless=True)
-                print(
-                    f"  1. ✅ Created BrowserConfig: headless={browser_config.headless}"
-                )
-
-                # Step 2: Get adapter
-                adapter = _get_browser_adapter(strategy, browser_config)
-                print(f"  2. ✅ Got adapter: {adapter.__class__.__name__}")
-
-                # Step 3: Test crawler creation
-                crawler = await get_crawler(browser_config, adapter)
-                print(f"  3. ✅ Created crawler: {crawler.__class__.__name__}")
-
-                # Step 4: Test the strategy inside the crawler
-                if hasattr(crawler, "crawler_strategy"):
-                    strategy_obj = crawler.crawler_strategy
-                    print(
-                        f"  4. ✅ Crawler strategy: {strategy_obj.__class__.__name__}"
-                    )
-
-                    if hasattr(strategy_obj, "adapter"):
-                        adapter_in_strategy = strategy_obj.adapter
-                        print(
-                            f"  5. ✅ Adapter in strategy: {adapter_in_strategy.__class__.__name__}"
-                        )
-
-                        # Check if it's the same adapter we passed
-                        if adapter_in_strategy.__class__ == adapter.__class__:
-                            print(f"  6. ✅ Adapter correctly passed through!")
-                        else:
-                            print(
-                                f"  6. ❌ Adapter mismatch! Expected {adapter.__class__.__name__}, got {adapter_in_strategy.__class__.__name__}"
-                            )
-                    else:
-                        print(f"  5. ❌ No adapter found in strategy")
-                else:
-                    print(f"  4. ❌ No crawler_strategy found in crawler")
-
-                # Step 5: Test actual crawling
-                test_html = (
-                    "<html><body><h1>Test</h1><p>Adapter test page</p></body></html>"
-                )
-                with open("/tmp/adapter_test.html", "w") as f:
-                    f.write(test_html)
-
-                crawler_config = CrawlerRunConfig(cache_mode="bypass")
-                result = await crawler.arun(
-                    url="file:///tmp/adapter_test.html", config=crawler_config
-                )
-
-                if result.success:
-                    print(
-                        f"  7. ✅ Crawling successful! Content length: {len(result.markdown)}"
-                    )
-                else:
-                    print(f"  7. ❌ Crawling failed: {result.error_message}")
-
-            except Exception as e:
-                print(f"  ❌ Error testing {strategy}: {e}")
-                import traceback
-
-                traceback.print_exc()
-
-        print(f"\n🎉 Adapter chain testing completed!")
-
-    except Exception as e:
-        print(f"❌ Setup error: {e}")
-        import traceback
-
-        traceback.print_exc()
-
-
-if __name__ == "__main__":
-    asyncio.run(test_adapter_chain())

tests/docker/extended_features/test_adapter_verification.py

@@ -1,128 +0,0 @@
-#!/usr/bin/env python3
-"""
-Test what's actually happening with the adapters - check the correct attribute
-"""
-
-import asyncio
-import os
-import sys
-
-import pytest
-
-# Add the project root to Python path
-sys.path.insert(0, os.getcwd())
-sys.path.insert(0, os.path.join(os.getcwd(), "deploy", "docker"))
-
-
-@pytest.mark.asyncio
-async def test_adapter_verification():
-    """Test that adapters are actually being used correctly"""
-    print("🔍 Testing Adapter Usage Verification")
-    print("=" * 50)
-
-    try:
-        # Import the API functions
-        from api import _apply_headless_setting, _get_browser_adapter
-        from crawler_pool import get_crawler
-
-        from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
-
-        print("✅ Successfully imported all functions")
-
-        # Test different strategies
-        strategies = [
-            ("default", "PlaywrightAdapter"),
-            ("stealth", "StealthAdapter"),
-            ("undetected", "UndetectedAdapter"),
-        ]
-
-        for strategy, expected_adapter in strategies:
-            print(f"\n🧪 Testing {strategy} strategy (expecting {expected_adapter}):")
-            print("-" * 50)
-
-            try:
-                # Step 1: Create browser config
-                browser_config = BrowserConfig(headless=True)
-                print(f"  1. ✅ Created BrowserConfig")
-
-                # Step 2: Get adapter
-                adapter = _get_browser_adapter(strategy, browser_config)
-                adapter_name = adapter.__class__.__name__
-                print(f"  2. ✅ Got adapter: {adapter_name}")
-
-                if adapter_name == expected_adapter:
-                    print(f"  3. ✅ Correct adapter type selected!")
-                else:
-                    print(
-                        f"  3. ❌ Wrong adapter! Expected {expected_adapter}, got {adapter_name}"
-                    )
-
-                # Step 4: Test crawler creation and adapter usage
-                crawler = await get_crawler(browser_config, adapter)
-                print(f"  4. ✅ Created crawler")
-
-                # Check if the strategy has the correct adapter
-                if hasattr(crawler, "crawler_strategy"):
-                    strategy_obj = crawler.crawler_strategy
-
-                    if hasattr(strategy_obj, "adapter"):
-                        adapter_in_strategy = strategy_obj.adapter
-                        strategy_adapter_name = adapter_in_strategy.__class__.__name__
-                        print(f"  5. ✅ Strategy adapter: {strategy_adapter_name}")
-
-                        # Check if it matches what we expected
-                        if strategy_adapter_name == expected_adapter:
-                            print(f"  6. ✅ ADAPTER CORRECTLY APPLIED!")
-                        else:
-                            print(
-                                f"  6. ❌ Adapter mismatch! Expected {expected_adapter}, strategy has {strategy_adapter_name}"
-                            )
-                    else:
-                        print(f"  5. ❌ No adapter attribute found in strategy")
-                else:
-                    print(f"  4. ❌ No crawler_strategy found in crawler")
-
-                # Test with a real website to see user-agent differences
-                print(f"  7. 🌐 Testing with httpbin.org...")
-
-                crawler_config = CrawlerRunConfig(cache_mode="bypass")
-                result = await crawler.arun(
-                    url="https://httpbin.org/user-agent", config=crawler_config
-                )
-
-                if result.success:
-                    print(f"  8. ✅ Crawling successful!")
-                    if "user-agent" in result.markdown.lower():
-                        # Extract user agent info
-                        lines = result.markdown.split("\\n")
-                        ua_line = [
-                            line for line in lines if "user-agent" in line.lower()
-                        ]
-                        if ua_line:
-                            print(f"  9. 🔍 User-Agent detected: {ua_line[0][:100]}...")
-                        else:
-                            print(f"  9. 📝 Content: {result.markdown[:200]}...")
-                    else:
-                        print(
-                            f"  9. 📝 No user-agent in content, got: {result.markdown[:100]}..."
-                        )
-                else:
-                    print(f"  8. ❌ Crawling failed: {result.error_message}")
-
-            except Exception as e:
-                print(f"  ❌ Error testing {strategy}: {e}")
-                import traceback
-
-                traceback.print_exc()
-
-        print(f"\n🎉 Adapter verification completed!")
-
-    except Exception as e:
-        print(f"❌ Setup error: {e}")
-        import traceback
-
-        traceback.print_exc()
-
-
-if __name__ == "__main__":
-    asyncio.run(test_adapter_verification())

tests/docker/extended_features/test_all_features.py

@@ -1,677 +0,0 @@
-#!/usr/bin/env python3
-"""
-Comprehensive Test Suite for Docker Extended Features
-Tests all advanced features: URL seeding, adaptive crawling, browser adapters,
-proxy rotation, and dispatchers.
-"""
-
-import asyncio
-import sys
-from pathlib import Path
-from typing import Any, Dict, List
-
-import aiohttp
-from rich import box
-from rich.console import Console
-from rich.panel import Panel
-from rich.table import Table
-
-# Configuration
-API_BASE_URL = "http://localhost:11235"
-console = Console()
-
-
-class TestResultData:
-    def __init__(self, name: str, category: str):
-        self.name = name
-        self.category = category
-        self.passed = False
-        self.error = None
-        self.duration = 0.0
-        self.details = {}
-
-
-class ExtendedFeaturesTestSuite:
-    def __init__(self, base_url: str = API_BASE_URL):
-        self.base_url = base_url
-        self.headers = {"Content-Type": "application/json"}
-        self.results: List[TestResultData] = []
-
-    async def check_server_health(self) -> bool:
-        """Check if the server is running"""
-        try:
-            async with aiohttp.ClientSession() as session:
-                async with session.get(
-                    f"{self.base_url}/health", timeout=aiohttp.ClientTimeout(total=5)
-                ) as response:
-                    return response.status == 200
-        except Exception as e:
-            console.print(f"[red]Server health check failed: {e}[/red]")
-            return False
-
-    # ========================================================================
-    # URL SEEDING TESTS
-    # ========================================================================
-
-    async def test_url_seeding_basic(self) -> TestResultData:
-        """Test basic URL seeding functionality"""
-        result = TestResultData("Basic URL Seeding", "URL Seeding")
-        try:
-            import time
-
-            start = time.time()
-
-            payload = {
-                "url": "https://www.nbcnews.com",
-                "config": {"max_urls": 10, "filter_type": "all"},
-            }
-
-            async with aiohttp.ClientSession() as session:
-                async with session.post(
-                    f"{self.base_url}/seed",
-                    headers=self.headers,
-                    json=payload,
-                    timeout=aiohttp.ClientTimeout(total=30),
-                ) as response:
-                    if response.status == 200:
-                        data = await response.json()
-                        # API returns: {"seed_url": [list of urls], "count": n}
-                        urls = data.get("seed_url", [])
-
-                        result.passed = len(urls) > 0
-                        result.details = {
-                            "urls_found": len(urls),
-                            "sample_url": urls[0] if urls else None,
-                        }
-                    else:
-                        result.error = f"Status {response.status}"
-
-            result.duration = time.time() - start
-        except Exception as e:
-            result.error = str(e)
-
-        return result
-
-    async def test_url_seeding_with_filters(self) -> TestResultData:
-        """Test URL seeding with different filter types"""
-        result = TestResultData("URL Seeding with Filters", "URL Seeding")
-        try:
-            import time
-
-            start = time.time()
-
-            payload = {
-                "url": "https://www.nbcnews.com",
-                "config": {
-                    "max_urls": 20,
-                    "filter_type": "domain",
-                    "exclude_external": True,
-                },
-            }
-
-            async with aiohttp.ClientSession() as session:
-                async with session.post(
-                    f"{self.base_url}/seed",
-                    headers=self.headers,
-                    json=payload,
-                    timeout=aiohttp.ClientTimeout(total=30),
-                ) as response:
-                    if response.status == 200:
-                        data = await response.json()
-                        # API returns: {"seed_url": [list of urls], "count": n}
-                        urls = data.get("seed_url", [])
-
-                        result.passed = len(urls) > 0
-                        result.details = {
-                            "urls_found": len(urls),
-                            "filter_type": "domain",
-                        }
-                    else:
-                        result.error = f"Status {response.status}"
-
-            result.duration = time.time() - start
-        except Exception as e:
-            result.error = str(e)
-
-        return result
-
-    # ========================================================================
-    # ADAPTIVE CRAWLING TESTS
-    # ========================================================================
-
-    async def test_adaptive_crawling_basic(self) -> TestResultData:
-        """Test basic adaptive crawling"""
-        result = TestResultData("Basic Adaptive Crawling", "Adaptive Crawling")
-        try:
-            import time
-
-            start = time.time()
-
-            payload = {
-                "urls": ["https://example.com"],
-                "browser_config": {"headless": True},
-                "crawler_config": {"adaptive": True, "adaptive_threshold": 0.5},
-            }
-
-            async with aiohttp.ClientSession() as session:
-                async with session.post(
-                    f"{self.base_url}/crawl",
-                    headers=self.headers,
-                    json=payload,
-                    timeout=aiohttp.ClientTimeout(total=60),
-                ) as response:
-                    if response.status == 200:
-                        data = await response.json()
-                        result.passed = data.get("success", False)
-                        result.details = {"results_count": len(data.get("results", []))}
-                    else:
-                        result.error = f"Status {response.status}"
-
-            result.duration = time.time() - start
-        except Exception as e:
-            result.error = str(e)
-
-        return result
-
-    async def test_adaptive_crawling_with_strategy(self) -> TestResultData:
-        """Test adaptive crawling with custom strategy"""
-        result = TestResultData("Adaptive Crawling with Strategy", "Adaptive Crawling")
-        try:
-            import time
-
-            start = time.time()
-
-            payload = {
-                "urls": ["https://httpbin.org/html"],
-                "browser_config": {"headless": True},
-                "crawler_config": {
-                    "adaptive": True,
-                    "adaptive_threshold": 0.7,
-                    "word_count_threshold": 10,
-                },
-            }
-
-            async with aiohttp.ClientSession() as session:
-                async with session.post(
-                    f"{self.base_url}/crawl",
-                    headers=self.headers,
-                    json=payload,
-                    timeout=aiohttp.ClientTimeout(total=60),
-                ) as response:
-                    if response.status == 200:
-                        data = await response.json()
-                        result.passed = data.get("success", False)
-                        result.details = {"adaptive_threshold": 0.7}
-                    else:
-                        result.error = f"Status {response.status}"
-
-            result.duration = time.time() - start
-        except Exception as e:
-            result.error = str(e)
-
-        return result
-
-    # ========================================================================
-    # BROWSER ADAPTER TESTS
-    # ========================================================================
-
-    async def test_browser_adapter_default(self) -> TestResultData:
-        """Test default browser adapter"""
-        result = TestResultData("Default Browser Adapter", "Browser Adapters")
-        try:
-            import time
-
-            start = time.time()
-
-            payload = {
-                "urls": ["https://example.com"],
-                "browser_config": {"headless": True},
-                "crawler_config": {},
-                "anti_bot_strategy": "default",
-            }
-
-            async with aiohttp.ClientSession() as session:
-                async with session.post(
-                    f"{self.base_url}/crawl",
-                    headers=self.headers,
-                    json=payload,
-                    timeout=aiohttp.ClientTimeout(total=60),
-                ) as response:
-                    if response.status == 200:
-                        data = await response.json()
-                        result.passed = data.get("success", False)
-                        result.details = {"adapter": "default"}
-                    else:
-                        result.error = f"Status {response.status}"
-
-            result.duration = time.time() - start
-        except Exception as e:
-            result.error = str(e)
-
-        return result
-
-    async def test_browser_adapter_stealth(self) -> TestResultData:
-        """Test stealth browser adapter"""
-        result = TestResultData("Stealth Browser Adapter", "Browser Adapters")
-        try:
-            import time
-
-            start = time.time()
-
-            payload = {
-                "urls": ["https://example.com"],
-                "browser_config": {"headless": True},
-                "crawler_config": {},
-                "anti_bot_strategy": "stealth",
-            }
-
-            async with aiohttp.ClientSession() as session:
-                async with session.post(
-                    f"{self.base_url}/crawl",
-                    headers=self.headers,
-                    json=payload,
-                    timeout=aiohttp.ClientTimeout(total=60),
-                ) as response:
-                    if response.status == 200:
-                        data = await response.json()
-                        result.passed = data.get("success", False)
-                        result.details = {"adapter": "stealth"}
-                    else:
-                        result.error = f"Status {response.status}"
-
-            result.duration = time.time() - start
-        except Exception as e:
-            result.error = str(e)
-
-        return result
-
-    async def test_browser_adapter_undetected(self) -> TestResultData:
-        """Test undetected browser adapter"""
-        result = TestResultData("Undetected Browser Adapter", "Browser Adapters")
-        try:
-            import time
-
-            start = time.time()
-
-            payload = {
-                "urls": ["https://example.com"],
-                "browser_config": {"headless": True},
-                "crawler_config": {},
-                "anti_bot_strategy": "undetected",
-            }
-
-            async with aiohttp.ClientSession() as session:
-                async with session.post(
-                    f"{self.base_url}/crawl",
-                    headers=self.headers,
-                    json=payload,
-                    timeout=aiohttp.ClientTimeout(total=60),
-                ) as response:
-                    if response.status == 200:
-                        data = await response.json()
-                        result.passed = data.get("success", False)
-                        result.details = {"adapter": "undetected"}
-                    else:
-                        result.error = f"Status {response.status}"
-
-            result.duration = time.time() - start
-        except Exception as e:
-            result.error = str(e)
-
-        return result
-
-    # ========================================================================
-    # PROXY ROTATION TESTS
-    # ========================================================================
-
-    async def test_proxy_rotation_round_robin(self) -> TestResultData:
-        """Test round robin proxy rotation"""
-        result = TestResultData("Round Robin Proxy Rotation", "Proxy Rotation")
-        try:
-            import time
-
-            start = time.time()
-
-            payload = {
-                "urls": ["https://httpbin.org/ip"],
-                "browser_config": {"headless": True},
-                "crawler_config": {},
-                "proxy_rotation_strategy": "round_robin",
-                "proxies": [
-                    {"server": "http://proxy1.example.com:8080"},
-                    {"server": "http://proxy2.example.com:8080"},
-                ],
-            }
-
-            async with aiohttp.ClientSession() as session:
-                async with session.post(
-                    f"{self.base_url}/crawl",
-                    headers=self.headers,
-                    json=payload,
-                    timeout=aiohttp.ClientTimeout(total=60),
-                ) as response:
-                    # This might fail due to invalid proxies, but we're testing the API accepts it
-                    result.passed = response.status in [
-                        200,
-                        500,
-                    ]  # Accept either success or expected failure
-                    result.details = {
-                        "strategy": "round_robin",
-                        "status": response.status,
-                    }
-
-            result.duration = time.time() - start
-        except Exception as e:
-            result.error = str(e)
-
-        return result
-
-    async def test_proxy_rotation_random(self) -> TestResultData:
-        """Test random proxy rotation"""
-        result = TestResultData("Random Proxy Rotation", "Proxy Rotation")
-        try:
-            import time
-
-            start = time.time()
-
-            payload = {
-                "urls": ["https://httpbin.org/ip"],
-                "browser_config": {"headless": True},
-                "crawler_config": {},
-                "proxy_rotation_strategy": "random",
-                "proxies": [
-                    {"server": "http://proxy1.example.com:8080"},
-                    {"server": "http://proxy2.example.com:8080"},
-                ],
-            }
-
-            async with aiohttp.ClientSession() as session:
-                async with session.post(
-                    f"{self.base_url}/crawl",
-                    headers=self.headers,
-                    json=payload,
-                    timeout=aiohttp.ClientTimeout(total=60),
-                ) as response:
-                    result.passed = response.status in [200, 500]
-                    result.details = {"strategy": "random", "status": response.status}
-
-            result.duration = time.time() - start
-        except Exception as e:
-            result.error = str(e)
-
-        return result
-
-    # ========================================================================
-    # DISPATCHER TESTS
-    # ========================================================================
-
-    async def test_dispatcher_memory_adaptive(self) -> TestResultData:
-        """Test memory adaptive dispatcher"""
-        result = TestResultData("Memory Adaptive Dispatcher", "Dispatchers")
-        try:
-            import time
-
-            start = time.time()
-
-            payload = {
-                "urls": ["https://example.com"],
-                "browser_config": {"headless": True},
-                "crawler_config": {"screenshot": True},
-                "dispatcher": "memory_adaptive",
-            }
-
-            async with aiohttp.ClientSession() as session:
-                async with session.post(
-                    f"{self.base_url}/crawl",
-                    headers=self.headers,
-                    json=payload,
-                    timeout=aiohttp.ClientTimeout(total=60),
-                ) as response:
-                    if response.status == 200:
-                        data = await response.json()
-                        result.passed = data.get("success", False)
-                        if result.passed and data.get("results"):
-                            has_screenshot = (
-                                data["results"][0].get("screenshot") is not None
-                            )
-                            result.details = {
-                                "dispatcher": "memory_adaptive",
-                                "screenshot_captured": has_screenshot,
-                            }
-                    else:
-                        result.error = f"Status {response.status}"
-
-            result.duration = time.time() - start
-        except Exception as e:
-            result.error = str(e)
-
-        return result
-
-    async def test_dispatcher_semaphore(self) -> TestResultData:
-        """Test semaphore dispatcher"""
-        result = TestResultData("Semaphore Dispatcher", "Dispatchers")
-        try:
-            import time
-
-            start = time.time()
-
-            payload = {
-                "urls": ["https://example.com"],
-                "browser_config": {"headless": True},
-                "crawler_config": {},
-                "dispatcher": "semaphore",
-            }
-
-            async with aiohttp.ClientSession() as session:
-                async with session.post(
-                    f"{self.base_url}/crawl",
-                    headers=self.headers,
-                    json=payload,
-                    timeout=aiohttp.ClientTimeout(total=60),
-                ) as response:
-                    if response.status == 200:
-                        data = await response.json()
-                        result.passed = data.get("success", False)
-                        result.details = {"dispatcher": "semaphore"}
-                    else:
-                        result.error = f"Status {response.status}"
-
-            result.duration = time.time() - start
-        except Exception as e:
-            result.error = str(e)
-
-        return result
-
-    async def test_dispatcher_endpoints(self) -> TestResultData:
-        """Test dispatcher management endpoints"""
-        result = TestResultData("Dispatcher Management Endpoints", "Dispatchers")
-        try:
-            import time
-
-            start = time.time()
-
-            async with aiohttp.ClientSession() as session:
-                # Test list dispatchers
-                async with session.get(
-                    f"{self.base_url}/dispatchers",
-                    headers=self.headers,
-                    timeout=aiohttp.ClientTimeout(total=10),
-                ) as response:
-                    if response.status == 200:
-                        data = await response.json()
-                        # API returns a list directly, not wrapped in a dict
-                        dispatchers = data if isinstance(data, list) else []
-                        result.passed = len(dispatchers) > 0
-                        result.details = {
-                            "dispatcher_count": len(dispatchers),
-                            "available": [d.get("type") for d in dispatchers],
-                        }
-                    else:
-                        result.error = f"Status {response.status}"
-
-            result.duration = time.time() - start
-        except Exception as e:
-            result.error = str(e)
-
-        return result
-
-    # ========================================================================
-    # TEST RUNNER
-    # ========================================================================
-
-    async def run_all_tests(self):
-        """Run all tests and collect results"""
-        console.print(
-            Panel.fit(
-                "[bold cyan]Extended Features Test Suite[/bold cyan]\n"
-                "Testing: URL Seeding, Adaptive Crawling, Browser Adapters, Proxy Rotation, Dispatchers",
-                border_style="cyan",
-            )
-        )
-
-        # Check server health first
-        console.print("\n[yellow]Checking server health...[/yellow]")
-        if not await self.check_server_health():
-            console.print(
-                "[red]❌ Server is not responding. Please start the Docker container.[/red]"
-            )
-            console.print(f"[yellow]Expected server at: {self.base_url}[/yellow]")
-            return
-
-        console.print("[green]✅ Server is healthy[/green]\n")
-
-        # Define all tests
-        tests = [
-            # URL Seeding
-            self.test_url_seeding_basic(),
-            self.test_url_seeding_with_filters(),
-            # Adaptive Crawling
-            self.test_adaptive_crawling_basic(),
-            self.test_adaptive_crawling_with_strategy(),
-            # Browser Adapters
-            self.test_browser_adapter_default(),
-            self.test_browser_adapter_stealth(),
-            self.test_browser_adapter_undetected(),
-            # Proxy Rotation
-            self.test_proxy_rotation_round_robin(),
-            self.test_proxy_rotation_random(),
-            # Dispatchers
-            self.test_dispatcher_memory_adaptive(),
-            self.test_dispatcher_semaphore(),
-            self.test_dispatcher_endpoints(),
-        ]
-
-        console.print(f"[cyan]Running {len(tests)} tests...[/cyan]\n")
-
-        # Run tests
-        for i, test_coro in enumerate(tests, 1):
-            console.print(f"[yellow]Running test {i}/{len(tests)}...[/yellow]")
-            test_result = await test_coro
-            self.results.append(test_result)
-
-            # Print immediate feedback
-            if test_result.passed:
-                console.print(
-                    f"[green]✅ {test_result.name} ({test_result.duration:.2f}s)[/green]"
-                )
-            else:
-                console.print(
-                    f"[red]❌ {test_result.name} ({test_result.duration:.2f}s)[/red]"
-                )
-                if test_result.error:
-                    console.print(f"   [red]Error: {test_result.error}[/red]")
-
-        # Display results
-        self.display_results()
-
-    def display_results(self):
-        """Display test results in a formatted table"""
-        console.print("\n")
-        console.print(
-            Panel.fit("[bold]Test Results Summary[/bold]", border_style="cyan")
-        )
-
-        # Group by category
-        categories = {}
-        for result in self.results:
-            if result.category not in categories:
-                categories[result.category] = []
-            categories[result.category].append(result)
-
-        # Display by category
-        for category, tests in categories.items():
-            table = Table(
-                title=f"\n{category}",
-                box=box.ROUNDED,
-                show_header=True,
-                header_style="bold cyan",
-            )
-            table.add_column("Test Name", style="white", width=40)
-            table.add_column("Status", style="white", width=10)
-            table.add_column("Duration", style="white", width=10)
-            table.add_column("Details", style="white", width=40)
-
-            for test in tests:
-                status = (
-                    "[green]✅ PASS[/green]" if test.passed else "[red]❌ FAIL[/red]"
-                )
-                duration = f"{test.duration:.2f}s"
-                details = str(test.details) if test.details else (test.error or "")
-                if test.error and len(test.error) > 40:
-                    details = test.error[:37] + "..."
-
-                table.add_row(test.name, status, duration, details)
-
-            console.print(table)
-
-        # Overall statistics
-        total_tests = len(self.results)
-        passed_tests = sum(1 for r in self.results if r.passed)
-        failed_tests = total_tests - passed_tests
-        pass_rate = (passed_tests / total_tests * 100) if total_tests > 0 else 0
-
-        console.print("\n")
-        stats_table = Table(box=box.DOUBLE, show_header=False, width=60)
-        stats_table.add_column("Metric", style="bold cyan", width=30)
-        stats_table.add_column("Value", style="bold white", width=30)
-
-        stats_table.add_row("Total Tests", str(total_tests))
-        stats_table.add_row("Passed", f"[green]{passed_tests}[/green]")
-        stats_table.add_row("Failed", f"[red]{failed_tests}[/red]")
-        stats_table.add_row("Pass Rate", f"[cyan]{pass_rate:.1f}%[/cyan]")
-
-        console.print(
-            Panel(
-                stats_table,
-                title="[bold]Overall Statistics[/bold]",
-                border_style="green" if pass_rate >= 80 else "yellow",
-            )
-        )
-
-        # Recommendations
-        if failed_tests > 0:
-            console.print(
-                "\n[yellow]💡 Some tests failed. Check the errors above for details.[/yellow]"
-            )
-            console.print("[yellow]   Common issues:[/yellow]")
-            console.print(
-                "[yellow]   - Server not fully started (wait ~30-40 seconds after docker compose up)[/yellow]"
-            )
-            console.print(
-                "[yellow]   - Invalid proxy servers in proxy rotation tests (expected)[/yellow]"
-            )
-            console.print("[yellow]   - Network connectivity issues[/yellow]")
-
-
-async def main():
-    """Main entry point"""
-    suite = ExtendedFeaturesTestSuite()
-    await suite.run_all_tests()
-
-
-if __name__ == "__main__":
-    try:
-        asyncio.run(main())
-    except KeyboardInterrupt:
-        console.print("\n[yellow]Tests interrupted by user[/yellow]")
-        sys.exit(1)

tests/docker/extended_features/test_anti_bot_strategy.py

@@ -1,172 +0,0 @@
-#!/usr/bin/env python3
-"""
-Test script for the anti_bot_strategy functionality in the FastAPI server.
-This script tests different browser adapter configurations.
-"""
-
-import json
-import time
-
-import requests
-
-# Test configurations for different anti_bot_strategy values
-test_configs = [
-    {
-        "name": "Default Strategy",
-        "payload": {
-            "urls": ["https://httpbin.org/user-agent"],
-            "anti_bot_strategy": "default",
-            "headless": True,
-            "browser_config": {},
-            "crawler_config": {},
-        },
-    },
-    {
-        "name": "Stealth Strategy",
-        "payload": {
-            "urls": ["https://httpbin.org/user-agent"],
-            "anti_bot_strategy": "stealth",
-            "headless": True,
-            "browser_config": {},
-            "crawler_config": {},
-        },
-    },
-    {
-        "name": "Undetected Strategy",
-        "payload": {
-            "urls": ["https://httpbin.org/user-agent"],
-            "anti_bot_strategy": "undetected",
-            "headless": True,
-            "browser_config": {},
-            "crawler_config": {},
-        },
-    },
-    {
-        "name": "Max Evasion Strategy",
-        "payload": {
-            "urls": ["https://httpbin.org/user-agent"],
-            "anti_bot_strategy": "max_evasion",
-            "headless": True,
-            "browser_config": {},
-            "crawler_config": {},
-        },
-    },
-]
-
-
-def test_api_endpoint(base_url="http://localhost:11235"):
-    """Test the crawl endpoint with different anti_bot_strategy values."""
-
-    print("🧪 Testing Anti-Bot Strategy API Implementation")
-    print("=" * 60)
-
-    # Check if server is running
-    try:
-        health_response = requests.get(f"{base_url}/health", timeout=5)
-        if health_response.status_code != 200:
-            print("❌ Server health check failed")
-            return False
-        print("✅ Server is running and healthy")
-    except requests.exceptions.RequestException as e:
-        print(f"❌ Cannot connect to server at {base_url}: {e}")
-        print(
-            "💡 Make sure the FastAPI server is running: python -m fastapi dev deploy/docker/server.py --port 11235"
-        )
-        return False
-
-    print()
-
-    # Test each configuration
-    for i, test_config in enumerate(test_configs, 1):
-        print(f"Test {i}: {test_config['name']}")
-        print("-" * 40)
-
-        try:
-            # Make request to crawl endpoint
-            response = requests.post(
-                f"{base_url}/crawl",
-                json=test_config["payload"],
-                headers={"Content-Type": "application/json"},
-                timeout=30,
-            )
-
-            if response.status_code == 200:
-                result = response.json()
-
-                # Check if crawl was successful
-                if result.get("results") and len(result["results"]) > 0:
-                    first_result = result["results"][0]
-                    if first_result.get("success"):
-                        print(f"✅ {test_config['name']} - SUCCESS")
-
-                        # Try to extract user agent info from response
-                        markdown_content = first_result.get("markdown", {})
-                        if isinstance(markdown_content, dict):
-                            # If markdown is a dict, look for raw_markdown
-                            markdown_text = markdown_content.get("raw_markdown", "")
-                        else:
-                            # If markdown is a string
-                            markdown_text = markdown_content or ""
-
-                        if "user-agent" in markdown_text.lower():
-                            print("  🕷️  User agent info found in response")
-
-                        print(f"  📄 Markdown length: {len(markdown_text)} characters")
-                    else:
-                        error_msg = first_result.get("error_message", "Unknown error")
-                        print(f"❌ {test_config['name']} - FAILED: {error_msg}")
-                else:
-                    print(f"❌ {test_config['name']} - No results returned")
-
-            else:
-                print(f"❌ {test_config['name']} - HTTP {response.status_code}")
-                print(f"  Response: {response.text[:200]}...")
-
-        except requests.exceptions.Timeout:
-            print(f"⏰ {test_config['name']} - TIMEOUT (30s)")
-        except requests.exceptions.RequestException as e:
-            print(f"❌ {test_config['name']} - REQUEST ERROR: {e}")
-        except Exception as e:
-            print(f"❌ {test_config['name']} - UNEXPECTED ERROR: {e}")
-
-        print()
-
-        # Brief pause between requests
-        time.sleep(1)
-
-    print("🏁 Testing completed!")
-
-
-def test_schema_validation():
-    """Test that the API accepts the new schema fields."""
-    print("📋 Testing Schema Validation")
-    print("-" * 30)
-
-    # Test payload with all new fields
-    test_payload = {
-        "urls": ["https://httpbin.org/headers"],
-        "anti_bot_strategy": "stealth",
-        "headless": False,
-        "browser_config": {
-            "headless": True  # This should be overridden by the top-level headless
-        },
-        "crawler_config": {},
-    }
-
-    print(
-        "✅ Schema validation: anti_bot_strategy and headless fields are properly defined"
-    )
-    print(f"✅ Test payload: {json.dumps(test_payload, indent=2)}")
-    print()
-
-
-if __name__ == "__main__":
-    print("🚀 Crawl4AI Anti-Bot Strategy Test Suite")
-    print("=" * 50)
-    print()
-
-    # Test schema first
-    test_schema_validation()
-
-    # Test API functionality
-    test_api_endpoint()

tests/docker/extended_features/test_antibot_simple.py

@@ -1,120 +0,0 @@
-#!/usr/bin/env python3
-"""
-Simple test of anti-bot strategy functionality
-"""
-
-import asyncio
-import os
-import sys
-
-import pytest
-
-# Add the project root to Python path
-sys.path.insert(0, os.getcwd())
-
-
-@pytest.mark.asyncio
-async def test_antibot_strategies():
-    """Test different anti-bot strategies"""
-    print("🧪 Testing Anti-Bot Strategies with AsyncWebCrawler")
-    print("=" * 60)
-
-    try:
-        from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
-        from crawl4ai.browser_adapter import PlaywrightAdapter
-
-        # Test HTML content
-        test_html = """
-        <html>
-        <head><title>Test Page</title></head>
-        <body>
-            <h1>Anti-Bot Strategy Test</h1>
-            <p>This page tests different browser adapters.</p>
-            <div id="content">
-                <p>User-Agent detection test</p>
-                <script>
-                    document.getElementById('content').innerHTML += 
-                        '<p>Browser: ' + navigator.userAgent + '</p>';
-                </script>
-            </div>
-        </body>
-        </html>
-        """
-
-        # Save test HTML
-        with open("/tmp/antibot_test.html", "w") as f:
-            f.write(test_html)
-
-        test_url = "file:///tmp/antibot_test.html"
-
-        strategies = [
-            ("default", "Default Playwright"),
-            ("stealth", "Stealth Mode"),
-        ]
-
-        for strategy, description in strategies:
-            print(f"\n🔍 Testing: {description} (strategy: {strategy})")
-            print("-" * 40)
-
-            try:
-                # Import adapter based on strategy
-                if strategy == "stealth":
-                    try:
-                        from crawl4ai import StealthAdapter
-
-                        adapter = StealthAdapter()
-                        print(f"✅ Using StealthAdapter")
-                    except ImportError:
-                        print(
-                            f"⚠️  StealthAdapter not available, using PlaywrightAdapter"
-                        )
-                        adapter = PlaywrightAdapter()
-                else:
-                    adapter = PlaywrightAdapter()
-                    print(f"✅ Using PlaywrightAdapter")
-
-                # Configure browser
-                browser_config = BrowserConfig(headless=True, browser_type="chromium")
-
-                # Configure crawler
-                crawler_config = CrawlerRunConfig(cache_mode="bypass")
-
-                # Run crawler
-                async with AsyncWebCrawler(
-                    config=browser_config, browser_adapter=adapter
-                ) as crawler:
-                    result = await crawler.arun(url=test_url, config=crawler_config)
-
-                    if result.success:
-                        print(f"✅ Crawl successful")
-                        print(f"   📄 Title: {result.metadata.get('title', 'N/A')}")
-                        print(f"   📏 Content length: {len(result.markdown)} chars")
-
-                        # Check if user agent info is in content
-                        if (
-                            "User-Agent" in result.markdown
-                            or "Browser:" in result.markdown
-                        ):
-                            print(f"   🔍 User-agent info detected in content")
-                        else:
-                            print(f"   ℹ️  No user-agent info in content")
-                    else:
-                        print(f"❌ Crawl failed: {result.error_message}")
-
-            except Exception as e:
-                print(f"❌ Error testing {strategy}: {e}")
-                import traceback
-
-                traceback.print_exc()
-
-        print(f"\n🎉 Anti-bot strategy testing completed!")
-
-    except Exception as e:
-        print(f"❌ Setup error: {e}")
-        import traceback
-
-        traceback.print_exc()
-
-
-if __name__ == "__main__":
-    asyncio.run(test_antibot_strategies())

tests/docker/extended_features/test_bot_detection.py

@@ -1,201 +0,0 @@
-#!/usr/bin/env python3
-"""
-Fixed version of test_bot_detection.py with proper timeouts and error handling
-"""
-
-import asyncio
-import os
-import sys
-import signal
-import logging
-from contextlib import asynccontextmanager
-
-import pytest
-
-# Add the project root to Python path
-sys.path.insert(0, os.getcwd())
-sys.path.insert(0, os.path.join(os.getcwd(), "deploy", "docker"))
-
-# Set up logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-# Global timeout handler
-class TimeoutError(Exception):
-    pass
-
-def timeout_handler(signum, frame):
-    raise TimeoutError("Operation timed out")
-
-@asynccontextmanager
-async def timeout_context(seconds):
-    """Context manager for timeout handling"""
-    try:
-        yield
-    except asyncio.TimeoutError:
-        logger.error(f"Operation timed out after {seconds} seconds")
-        raise
-    except TimeoutError:
-        logger.error(f"Operation timed out after {seconds} seconds")
-        raise
-
-async def safe_crawl_with_timeout(crawler, url, config, timeout_seconds=30):
-    """Safely crawl a URL with timeout"""
-    try:
-        # Use asyncio.wait_for to add timeout
-        result = await asyncio.wait_for(
-            crawler.arun(url=url, config=config),
-            timeout=timeout_seconds
-        )
-        return result
-    except asyncio.TimeoutError:
-        logger.error(f"Crawl timed out for {url} after {timeout_seconds} seconds")
-        return None
-    except Exception as e:
-        logger.error(f"Crawl failed for {url}: {e}")
-        return None
-
-@pytest.mark.asyncio
-async def test_bot_detection():
-    """Test adapters against bot detection with proper timeouts"""
-    print("🤖 Testing Adapters Against Bot Detection (Fixed Version)")
-    print("=" * 60)
-
-    # Set global timeout for the entire test (5 minutes)
-    test_timeout = 300
-    original_handler = signal.signal(signal.SIGALRM, timeout_handler)
-    signal.alarm(test_timeout)
-
-    crawlers_to_cleanup = []
-
-    try:
-        from api import _get_browser_adapter
-        from crawler_pool import get_crawler
-        from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
-
-        # Test with a site that detects automation
-        test_sites = [
-            "https://bot.sannysoft.com/",  # Bot detection test site
-            "https://httpbin.org/headers",  # Headers inspection
-        ]
-
-        strategies = [
-            ("default", "PlaywrightAdapter"),
-            ("stealth", "StealthAdapter"),
-            ("undetected", "UndetectedAdapter"),
-        ]
-
-        # Test with smaller browser config to reduce resource usage
-        browser_config = BrowserConfig(
-            headless=True,
-            verbose=False,
-            viewport_width=1024,
-            viewport_height=768
-        )
-
-        for site in test_sites:
-            print(f"\n🌐 Testing site: {site}")
-            print("=" * 60)
-
-            for strategy, expected_adapter in strategies:
-                print(f"\n  🧪 {strategy} strategy:")
-                print(f"  {'-' * 30}")
-
-                try:
-                    # Get adapter with timeout
-                    adapter = _get_browser_adapter(strategy, browser_config)
-                    print(f"    ✅ Using {adapter.__class__.__name__}")
-
-                    # Get crawler with timeout
-                    try:
-                        crawler = await asyncio.wait_for(
-                            get_crawler(browser_config, adapter),
-                            timeout=20  # 20 seconds timeout for crawler creation
-                        )
-                        crawlers_to_cleanup.append(crawler)
-                        print(f"    ✅ Crawler created successfully")
-                    except asyncio.TimeoutError:
-                        print(f"    ❌ Crawler creation timed out")
-                        continue
-
-                    # Crawl with timeout
-                    crawler_config = CrawlerRunConfig(
-                        cache_mode="bypass",
-                        wait_until="domcontentloaded",  # Faster than networkidle
-                        word_count_threshold=5  # Lower threshold for faster processing
-                    )
-
-                    result = await safe_crawl_with_timeout(
-                        crawler, site, crawler_config, timeout_seconds=20
-                    )
-
-                    if result and result.success:
-                        content = result.markdown[:500] if result.markdown else ""
-                        print(f"    ✅ Crawl successful ({len(result.markdown) if result.markdown else 0} chars)")
-
-                        # Look for bot detection indicators
-                        bot_indicators = [
-                            "webdriver",
-                            "automation",
-                            "bot detected",
-                            "chrome-devtools",
-                            "headless",
-                            "selenium",
-                        ]
-
-                        detected_indicators = []
-                        for indicator in bot_indicators:
-                            if indicator.lower() in content.lower():
-                                detected_indicators.append(indicator)
-
-                        if detected_indicators:
-                            print(f"    ⚠️  Detected indicators: {', '.join(detected_indicators)}")
-                        else:
-                            print(f"    ✅ No bot detection indicators found")
-
-                        # Show a snippet of content
-                        print(f"    📝 Content sample: {content[:200]}...")
-
-                    else:
-                        error_msg = result.error_message if result and hasattr(result, 'error_message') else "Unknown error"
-                        print(f"    ❌ Crawl failed: {error_msg}")
-
-                except asyncio.TimeoutError:
-                    print(f"    ❌ Strategy {strategy} timed out")
-                except Exception as e:
-                    print(f"    ❌ Error with {strategy} strategy: {e}")
-
-        print(f"\n🎉 Bot detection testing completed!")
-
-    except TimeoutError:
-        print(f"\n⏰ Test timed out after {test_timeout} seconds")
-        raise
-    except Exception as e:
-        print(f"❌ Setup error: {e}")
-        import traceback
-        traceback.print_exc()
-        raise
-    finally:
-        # Restore original signal handler
-        signal.alarm(0)
-        signal.signal(signal.SIGALRM, original_handler)
-
-        # Cleanup crawlers
-        print("\n🧹 Cleaning up browser instances...")
-        cleanup_tasks = []
-        for crawler in crawlers_to_cleanup:
-            if hasattr(crawler, 'close'):
-                cleanup_tasks.append(crawler.close())
-
-        if cleanup_tasks:
-            try:
-                await asyncio.wait_for(
-                    asyncio.gather(*cleanup_tasks, return_exceptions=True),
-                    timeout=10
-                )
-                print("✅ Cleanup completed")
-            except asyncio.TimeoutError:
-                print("⚠️  Cleanup timed out, but test completed")
-
-if __name__ == "__main__":
-    asyncio.run(test_bot_detection())

tests/docker/extended_features/test_final_summary.py

@@ -1,222 +0,0 @@
-#!/usr/bin/env python3
-"""
-Final Test Summary: Anti-Bot Strategy Implementation
-
-This script runs all the tests and provides a comprehensive summary
-of the anti-bot strategy implementation.
-"""
-
-import os
-import sys
-import time
-
-import requests
-
-# Add current directory to path for imports
-sys.path.insert(0, os.getcwd())
-sys.path.insert(0, os.path.join(os.getcwd(), "deploy", "docker"))
-
-
-def test_health():
-    """Test if the API server is running"""
-    try:
-        response = requests.get("http://localhost:11235/health", timeout=5)
-        assert response.status_code == 200, (
-            f"Server returned status {response.status_code}"
-        )
-    except Exception as e:
-        assert False, f"Cannot connect to server: {e}"
-
-
-def test_strategy_default():
-    """Test default anti-bot strategy"""
-    test_strategy_impl("default", "https://httpbin.org/headers")
-
-
-def test_strategy_stealth():
-    """Test stealth anti-bot strategy"""
-    test_strategy_impl("stealth", "https://httpbin.org/headers")
-
-
-def test_strategy_undetected():
-    """Test undetected anti-bot strategy"""
-    test_strategy_impl("undetected", "https://httpbin.org/headers")
-
-
-def test_strategy_max_evasion():
-    """Test max evasion anti-bot strategy"""
-    test_strategy_impl("max_evasion", "https://httpbin.org/headers")
-
-
-def test_strategy_impl(strategy_name, url="https://httpbin.org/headers"):
-    """Test a specific anti-bot strategy"""
-    try:
-        payload = {
-            "urls": [url],
-            "anti_bot_strategy": strategy_name,
-            "headless": True,
-            "browser_config": {},
-            "crawler_config": {},
-        }
-
-        response = requests.post(
-            "http://localhost:11235/crawl", json=payload, timeout=30
-        )
-
-        if response.status_code == 200:
-            data = response.json()
-            if data.get("success"):
-                assert True, f"Strategy {strategy_name} succeeded"
-            else:
-                assert False, f"API returned success=false for {strategy_name}"
-        else:
-            assert False, f"HTTP {response.status_code} for {strategy_name}"
-
-    except requests.exceptions.Timeout:
-        assert False, f"Timeout (30s) for {strategy_name}"
-    except Exception as e:
-        assert False, f"Error testing {strategy_name}: {e}"
-
-
-def test_core_functions():
-    """Test core adapter selection functions"""
-    try:
-        from api import _apply_headless_setting, _get_browser_adapter
-
-        from crawl4ai.async_configs import BrowserConfig
-
-        # Test adapter selection
-        config = BrowserConfig(headless=True)
-        strategies = ["default", "stealth", "undetected", "max_evasion"]
-        expected = [
-            "PlaywrightAdapter",
-            "StealthAdapter",
-            "UndetectedAdapter",
-            "UndetectedAdapter",
-        ]
-
-        for strategy, expected_adapter in zip(strategies, expected):
-            adapter = _get_browser_adapter(strategy, config)
-            actual = adapter.__class__.__name__
-            assert actual == expected_adapter, (
-                f"Expected {expected_adapter}, got {actual} for strategy {strategy}"
-            )
-
-    except Exception as e:
-        assert False, f"Core functions failed: {e}"
-
-
-def main():
-    """Run comprehensive test summary"""
-    print("🚀 Anti-Bot Strategy Implementation - Final Test Summary")
-    print("=" * 70)
-
-    # Test 1: Health Check
-    print("\n1️⃣  Server Health Check")
-    print("-" * 30)
-    if test_health():
-        print("✅ API server is running and healthy")
-    else:
-        print("❌ API server is not responding")
-        print(
-            "💡 Start server with: python -m fastapi dev deploy/docker/server.py --port 11235"
-        )
-        return
-
-    # Test 2: Core Functions
-    print("\n2️⃣  Core Function Testing")
-    print("-" * 30)
-    core_success, core_result = test_core_functions()
-    if core_success:
-        print("✅ Core adapter selection functions working:")
-        for strategy, expected, actual, match in core_result:
-            status = "✅" if match else "❌"
-            print(f"   {status} {strategy}: {actual} ({'✓' if match else '✗'})")
-    else:
-        print(f"❌ Core functions failed: {core_result}")
-
-    # Test 3: API Strategy Testing
-    print("\n3️⃣  API Strategy Testing")
-    print("-" * 30)
-    strategies = ["default", "stealth", "undetected", "max_evasion"]
-    all_passed = True
-
-    for strategy in strategies:
-        print(f"   Testing {strategy}...", end=" ")
-        success, message = test_strategy(strategy)
-        if success:
-            print("✅")
-        else:
-            print(f"❌ {message}")
-            all_passed = False
-
-    # Test 4: Different Scenarios
-    print("\n4️⃣  Scenario Testing")
-    print("-" * 30)
-
-    scenarios = [
-        ("Headers inspection", "stealth", "https://httpbin.org/headers"),
-        ("User-agent detection", "undetected", "https://httpbin.org/user-agent"),
-        ("HTML content", "default", "https://httpbin.org/html"),
-    ]
-
-    for scenario_name, strategy, url in scenarios:
-        print(f"   {scenario_name} ({strategy})...", end=" ")
-        success, message = test_strategy(strategy, url)
-        if success:
-            print("✅")
-        else:
-            print(f"❌ {message}")
-
-    # Summary
-    print("\n" + "=" * 70)
-    print("📋 IMPLEMENTATION SUMMARY")
-    print("=" * 70)
-
-    print("\n✅ COMPLETED FEATURES:")
-    print(
-        "   • Browser adapter selection (PlaywrightAdapter, StealthAdapter, UndetectedAdapter)"
-    )
-    print(
-        "   • API endpoints (/crawl and /crawl/stream) with anti_bot_strategy parameter"
-    )
-    print("   • Headless mode override functionality")
-    print("   • Crawler pool integration with adapter awareness")
-    print("   • Error handling and fallback mechanisms")
-    print("   • Comprehensive documentation and examples")
-
-    print("\n🎯 AVAILABLE STRATEGIES:")
-    print("   • default: PlaywrightAdapter - Fast, basic crawling")
-    print("   • stealth: StealthAdapter - Medium protection bypass")
-    print("   • undetected: UndetectedAdapter - High protection bypass")
-    print("   • max_evasion: UndetectedAdapter - Maximum evasion features")
-
-    print("\n🧪 TESTING STATUS:")
-    print("   ✅ Core functionality tests passing")
-    print("   ✅ API endpoint tests passing")
-    print("   ✅ Real website crawling working")
-    print("   ✅ All adapter strategies functional")
-    print("   ✅ Documentation and examples complete")
-
-    print("\n📚 DOCUMENTATION:")
-    print("   • ANTI_BOT_STRATEGY_DOCS.md - Complete API documentation")
-    print("   • ANTI_BOT_QUICK_REF.md - Quick reference guide")
-    print("   • examples_antibot_usage.py - Practical examples")
-    print("   • ANTI_BOT_README.md - Overview and getting started")
-
-    print("\n🚀 READY FOR PRODUCTION!")
-    print("\n💡 Usage example:")
-    print('   curl -X POST "http://localhost:11235/crawl" \\')
-    print('     -H "Content-Type: application/json" \\')
-    print('     -d \'{"urls":["https://example.com"],"anti_bot_strategy":"stealth"}\'')
-
-    print("\n" + "=" * 70)
-    if all_passed:
-        print("🎉 ALL TESTS PASSED - IMPLEMENTATION SUCCESSFUL! 🎉")
-    else:
-        print("⚠️  Some tests failed - check details above")
-    print("=" * 70)
-
-
-if __name__ == "__main__":
-    main()

tests/docker/extended_features/test_monitoring_quick.py

@@ -1,88 +0,0 @@
-#!/usr/bin/env python3
-"""
-Quick test to verify monitoring endpoints are working
-"""
-import requests
-import sys
-
-BASE_URL = "http://localhost:11234"
-
-def test_health():
-    """Test health endpoint"""
-    try:
-        response = requests.get(f"{BASE_URL}/monitoring/health", timeout=5)
-        if response.status_code == 200:
-            print("✅ Health check: PASSED")
-            print(f"   Response: {response.json()}")
-            return True
-        else:
-            print(f"❌ Health check: FAILED (status {response.status_code})")
-            return False
-    except Exception as e:
-        print(f"❌ Health check: ERROR - {e}")
-        return False
-
-def test_stats():
-    """Test stats endpoint"""
-    try:
-        response = requests.get(f"{BASE_URL}/monitoring/stats", timeout=5)
-        if response.status_code == 200:
-            stats = response.json()
-            print("✅ Stats endpoint: PASSED")
-            print(f"   Active crawls: {stats.get('active_crawls', 'N/A')}")
-            print(f"   Total crawls: {stats.get('total_crawls', 'N/A')}")
-            return True
-        else:
-            print(f"❌ Stats endpoint: FAILED (status {response.status_code})")
-            return False
-    except Exception as e:
-        print(f"❌ Stats endpoint: ERROR - {e}")
-        return False
-
-def test_url_stats():
-    """Test URL stats endpoint"""
-    try:
-        response = requests.get(f"{BASE_URL}/monitoring/stats/urls", timeout=5)
-        if response.status_code == 200:
-            print("✅ URL stats endpoint: PASSED")
-            url_stats = response.json()
-            print(f"   URLs tracked: {len(url_stats)}")
-            return True
-        else:
-            print(f"❌ URL stats endpoint: FAILED (status {response.status_code})")
-            return False
-    except Exception as e:
-        print(f"❌ URL stats endpoint: ERROR - {e}")
-        return False
-
-def main():
-    print("=" * 60)
-    print("Monitoring Endpoints Quick Test")
-    print("=" * 60)
-    print(f"\nTesting server at: {BASE_URL}")
-    print("\nMake sure the server is running:")
-    print("  cd deploy/docker && python server.py")
-    print("\n" + "-" * 60 + "\n")
-    
-    results = []
-    results.append(test_health())
-    print()
-    results.append(test_stats())
-    print()
-    results.append(test_url_stats())
-    
-    print("\n" + "=" * 60)
-    passed = sum(results)
-    total = len(results)
-    
-    if passed == total:
-        print(f"✅ All tests passed! ({passed}/{total})")
-        print("\nMonitoring endpoints are working correctly! 🎉")
-        return 0
-    else:
-        print(f"❌ Some tests failed ({passed}/{total} passed)")
-        print("\nPlease check the server logs for errors.")
-        return 1
-
-if __name__ == "__main__":
-    sys.exit(main())

tests/docker/test_hooks_utility.py

@@ -0,0 +1,193 @@
+"""
+Test script demonstrating the hooks_to_string utility and Docker client integration.
+"""
+import asyncio
+from crawl4ai import Crawl4aiDockerClient, hooks_to_string
+
+
+# Define hook functions as regular Python functions
+async def auth_hook(page, context, **kwargs):
+    """Add authentication cookies."""
+    await context.add_cookies([{
+        'name': 'test_cookie',
+        'value': 'test_value',
+        'domain': '.httpbin.org',
+        'path': '/'
+    }])
+    return page
+
+
+async def scroll_hook(page, context, **kwargs):
+    """Scroll to load lazy content."""
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    await page.wait_for_timeout(1000)
+    return page
+
+
+async def viewport_hook(page, context, **kwargs):
+    """Set custom viewport."""
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    return page
+
+
+async def test_hooks_utility():
+    """Test the hooks_to_string utility function."""
+    print("=" * 60)
+    print("Testing hooks_to_string utility")
+    print("=" * 60)
+
+    # Create hooks dictionary with function objects
+    hooks_dict = {
+        "on_page_context_created": auth_hook,
+        "before_retrieve_html": scroll_hook
+    }
+
+    # Convert to string format
+    hooks_string = hooks_to_string(hooks_dict)
+
+    print("\n✓ Successfully converted function objects to strings")
+    print(f"\n✓ Converted {len(hooks_string)} hooks:")
+    for hook_name in hooks_string.keys():
+        print(f"  - {hook_name}")
+
+    print("\n✓ Preview of converted hook:")
+    print("-" * 60)
+    print(hooks_string["on_page_context_created"][:200] + "...")
+    print("-" * 60)
+
+    return hooks_string
+
+
+async def test_docker_client_with_functions():
+    """Test Docker client with function objects (automatic conversion)."""
+    print("\n" + "=" * 60)
+    print("Testing Docker Client with Function Objects")
+    print("=" * 60)
+
+    # Note: This requires a running Crawl4AI Docker server
+    # Uncomment the following to test with actual server:
+
+    async with Crawl4aiDockerClient(base_url="http://localhost:11234", verbose=True) as client:
+        # Pass function objects directly - they'll be converted automatically
+        result = await client.crawl(
+            ["https://httpbin.org/html"],
+            hooks={
+                "on_page_context_created": auth_hook,
+                "before_retrieve_html": scroll_hook
+            },
+            hooks_timeout=30
+        )
+        print(f"\n✓ Crawl successful: {result.success}")
+        print(f"✓ URL: {result.url}")
+
+    print("\n✓ Docker client accepts function objects directly")
+    print("✓ Automatic conversion happens internally")
+    print("✓ No manual string formatting needed!")
+
+
+async def test_docker_client_with_strings():
+    """Test Docker client with pre-converted strings."""
+    print("\n" + "=" * 60)
+    print("Testing Docker Client with String Hooks")
+    print("=" * 60)
+
+    # Convert hooks to strings first
+    hooks_dict = {
+        "on_page_context_created": viewport_hook,
+        "before_retrieve_html": scroll_hook
+    }
+    hooks_string = hooks_to_string(hooks_dict)
+
+    # Note: This requires a running Crawl4AI Docker server
+    # Uncomment the following to test with actual server:
+
+    async with Crawl4aiDockerClient(base_url="http://localhost:11234", verbose=True) as client:
+        # Pass string hooks - they'll be used as-is
+        result = await client.crawl(
+            ["https://httpbin.org/html"],
+            hooks=hooks_string,
+            hooks_timeout=30
+        )
+        print(f"\n✓ Crawl successful: {result.success}")
+
+    print("\n✓ Docker client also accepts pre-converted strings")
+    print("✓ Backward compatible with existing code")
+
+
+async def show_usage_patterns():
+    """Show different usage patterns."""
+    print("\n" + "=" * 60)
+    print("Usage Patterns")
+    print("=" * 60)
+
+    print("\n1. Direct function usage (simplest):")
+    print("-" * 60)
+    print("""
+    async def my_hook(page, context, **kwargs):
+        await page.set_viewport_size({"width": 1920, "height": 1080})
+        return page
+
+    result = await client.crawl(
+        ["https://example.com"],
+        hooks={"on_page_context_created": my_hook}
+    )
+    """)
+
+    print("\n2. Convert then use:")
+    print("-" * 60)
+    print("""
+    hooks_dict = {"on_page_context_created": my_hook}
+    hooks_string = hooks_to_string(hooks_dict)
+
+    result = await client.crawl(
+        ["https://example.com"],
+        hooks=hooks_string
+    )
+    """)
+
+    print("\n3. Manual string (backward compatible):")
+    print("-" * 60)
+    print("""
+    hooks_string = {
+        "on_page_context_created": '''
+async def hook(page, context, **kwargs):
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    return page
+'''
+    }
+
+    result = await client.crawl(
+        ["https://example.com"],
+        hooks=hooks_string
+    )
+    """)
+
+
+async def main():
+    """Run all tests."""
+    print("\n🚀 Crawl4AI Hooks Utility Test Suite\n")
+
+    # Test the utility function
+    # await test_hooks_utility()
+
+    # Show usage with Docker client
+    # await test_docker_client_with_functions()
+    await test_docker_client_with_strings()
+
+    # Show different patterns
+    # await show_usage_patterns()
+
+    # print("\n" + "=" * 60)
+    # print("✓ All tests completed successfully!")
+    # print("=" * 60)
+    # print("\nKey Benefits:")
+    # print("  • Write hooks as regular Python functions")
+    # print("  • IDE support with autocomplete and type checking")
+    # print("  • Automatic conversion to API format")
+    # print("  • Backward compatible with string hooks")
+    # print("  • Same utility used everywhere")
+    # print("\n")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/docker/test_monitoring_endpoints.py

@@ -1,522 +0,0 @@
-"""
-Integration tests for monitoring and profiling endpoints.
-
-Tests all monitoring endpoints including profiling sessions, statistics,
-health checks, and real-time streaming.
-"""
-
-import asyncio
-import json
-import time
-from typing import Dict, List
-
-import pytest
-from httpx import AsyncClient
-
-# Base URL for the Docker API server
-BASE_URL = "http://localhost:11235"
-
-
-@pytest.fixture(scope="module")
-def event_loop():
-    """Create event loop for async tests."""
-    loop = asyncio.get_event_loop_policy().new_event_loop()
-    yield loop
-    loop.close()
-
-
-@pytest.fixture(scope="module")
-async def client():
-    """Create HTTP client for tests."""
-    async with AsyncClient(base_url=BASE_URL, timeout=60.0) as client:
-        yield client
-
-
-class TestHealthEndpoint:
-    """Tests for /monitoring/health endpoint."""
-
-    @pytest.mark.asyncio
-    async def test_health_check(self, client: AsyncClient):
-        """Test basic health check returns OK."""
-        response = await client.get("/monitoring/health")
-        assert response.status_code == 200
-        data = response.json()
-        assert data["status"] == "healthy"
-        assert "uptime_seconds" in data
-        assert data["uptime_seconds"] >= 0
-
-
-class TestStatsEndpoints:
-    """Tests for /monitoring/stats/* endpoints."""
-
-    @pytest.mark.asyncio
-    async def test_get_stats_empty(self, client: AsyncClient):
-        """Test getting stats when no crawls have been performed."""
-        # Reset stats first
-        await client.post("/monitoring/stats/reset")
-        
-        response = await client.get("/monitoring/stats")
-        assert response.status_code == 200
-        data = response.json()
-        
-        # Verify all expected fields
-        assert "active_crawls" in data
-        assert "total_crawls" in data
-        assert "successful_crawls" in data
-        assert "failed_crawls" in data
-        assert "success_rate" in data
-        assert "avg_duration_ms" in data
-        assert "total_bytes_processed" in data
-        assert "system_stats" in data
-        
-        # Verify system stats
-        system = data["system_stats"]
-        assert "cpu_percent" in system
-        assert "memory_percent" in system
-        assert "memory_used_mb" in system
-        assert "memory_available_mb" in system
-        assert "disk_usage_percent" in system
-        assert "active_processes" in system
-
-    @pytest.mark.asyncio
-    async def test_stats_after_crawl(self, client: AsyncClient):
-        """Test stats are updated after performing a crawl."""
-        # Reset stats
-        await client.post("/monitoring/stats/reset")
-        
-        # Perform a simple crawl
-        crawl_request = {
-            "urls": ["https://www.example.com"],
-            "crawler_config": {
-                "word_count_threshold": 10
-            }
-        }
-        crawl_response = await client.post("/crawl", json=crawl_request)
-        assert crawl_response.status_code == 200
-        
-        # Get stats
-        response = await client.get("/monitoring/stats")
-        assert response.status_code == 200
-        data = response.json()
-        
-        # Verify stats are updated
-        assert data["total_crawls"] >= 1
-        assert data["successful_crawls"] >= 0
-        assert data["failed_crawls"] >= 0
-        assert data["total_crawls"] == data["successful_crawls"] + data["failed_crawls"]
-        
-        # Verify success rate calculation
-        if data["total_crawls"] > 0:
-            expected_rate = (data["successful_crawls"] / data["total_crawls"]) * 100
-            assert abs(data["success_rate"] - expected_rate) < 0.01
-
-    @pytest.mark.asyncio
-    async def test_stats_reset(self, client: AsyncClient):
-        """Test resetting stats clears all counters."""
-        # Ensure we have some stats
-        crawl_request = {
-            "urls": ["https://www.example.com"],
-            "crawler_config": {"word_count_threshold": 10}
-        }
-        await client.post("/crawl", json=crawl_request)
-        
-        # Reset stats
-        reset_response = await client.post("/monitoring/stats/reset")
-        assert reset_response.status_code == 200
-        data = reset_response.json()
-        assert data["status"] == "reset"
-        assert "previous_stats" in data
-        
-        # Verify stats are cleared
-        stats_response = await client.get("/monitoring/stats")
-        stats = stats_response.json()
-        assert stats["total_crawls"] == 0
-        assert stats["successful_crawls"] == 0
-        assert stats["failed_crawls"] == 0
-        assert stats["active_crawls"] == 0
-
-    @pytest.mark.asyncio
-    async def test_url_specific_stats(self, client: AsyncClient):
-        """Test getting URL-specific statistics."""
-        # Reset and crawl
-        await client.post("/monitoring/stats/reset")
-        crawl_request = {
-            "urls": ["https://www.example.com"],
-            "crawler_config": {"word_count_threshold": 10}
-        }
-        await client.post("/crawl", json=crawl_request)
-        
-        # Get URL stats
-        response = await client.get("/monitoring/stats/urls")
-        assert response.status_code == 200
-        data = response.json()
-        
-        assert isinstance(data, list)
-        if len(data) > 0:
-            url_stat = data[0]
-            assert "url" in url_stat
-            assert "total_requests" in url_stat
-            assert "successful_requests" in url_stat
-            assert "failed_requests" in url_stat
-            assert "avg_duration_ms" in url_stat
-            assert "total_bytes_processed" in url_stat
-            assert "last_request_time" in url_stat
-
-
-class TestStatsStreaming:
-    """Tests for /monitoring/stats/stream SSE endpoint."""
-
-    @pytest.mark.asyncio
-    async def test_stats_stream_basic(self, client: AsyncClient):
-        """Test SSE streaming of statistics."""
-        # Start streaming (collect a few events then stop)
-        events = []
-        async with client.stream("GET", "/monitoring/stats/stream") as response:
-            assert response.status_code == 200
-            assert "text/event-stream" in response.headers.get("content-type", "")
-            
-            # Collect first 3 events
-            count = 0
-            async for line in response.aiter_lines():
-                if line.startswith("data: "):
-                    data_str = line[6:]  # Remove "data: " prefix
-                    data = json.loads(data_str)
-                    events.append(data)
-                    count += 1
-                    if count >= 3:
-                        break
-        
-        # Verify we got events
-        assert len(events) >= 3
-        
-        # Verify event structure
-        for event in events:
-            assert "active_crawls" in event
-            assert "total_crawls" in event
-            assert "successful_crawls" in event
-            assert "system_stats" in event
-
-    @pytest.mark.asyncio
-    async def test_stats_stream_during_crawl(self, client: AsyncClient):
-        """Test streaming updates during active crawl."""
-        # Start streaming in background
-        stream_task = None
-        events = []
-        
-        async def collect_stream():
-            async with client.stream("GET", "/monitoring/stats/stream") as response:
-                async for line in response.aiter_lines():
-                    if line.startswith("data: "):
-                        data_str = line[6:]
-                        data = json.loads(data_str)
-                        events.append(data)
-                        if len(events) >= 5:
-                            break
-        
-        # Start stream collection
-        stream_task = asyncio.create_task(collect_stream())
-        
-        # Wait a bit then start crawl
-        await asyncio.sleep(1)
-        crawl_request = {
-            "urls": ["https://www.example.com"],
-            "crawler_config": {"word_count_threshold": 10}
-        }
-        asyncio.create_task(client.post("/crawl", json=crawl_request))
-        
-        # Wait for events
-        try:
-            await asyncio.wait_for(stream_task, timeout=15.0)
-        except asyncio.TimeoutError:
-            stream_task.cancel()
-        
-        # Should have collected some events
-        assert len(events) > 0
-
-
-class TestProfilingEndpoints:
-    """Tests for /monitoring/profile/* endpoints."""
-
-    @pytest.mark.asyncio
-    async def test_list_profiling_sessions_empty(self, client: AsyncClient):
-        """Test listing profiling sessions when none exist."""
-        response = await client.get("/monitoring/profile")
-        assert response.status_code == 200
-        data = response.json()
-        assert "sessions" in data
-        assert isinstance(data["sessions"], list)
-
-    @pytest.mark.asyncio
-    async def test_start_profiling_session(self, client: AsyncClient):
-        """Test starting a new profiling session."""
-        request_data = {
-            "urls": ["https://www.example.com", "https://www.python.org"],
-            "duration_seconds": 2,
-            "crawler_config": {
-                "word_count_threshold": 10
-            }
-        }
-        
-        response = await client.post("/monitoring/profile/start", json=request_data)
-        assert response.status_code == 200
-        data = response.json()
-        
-        assert "session_id" in data
-        assert "status" in data
-        assert data["status"] == "running"
-        assert "started_at" in data
-        assert "urls" in data
-        assert len(data["urls"]) == 2
-        
-        return data["session_id"]
-
-    @pytest.mark.asyncio
-    async def test_get_profiling_session(self, client: AsyncClient):
-        """Test retrieving a profiling session by ID."""
-        # Start a session
-        request_data = {
-            "urls": ["https://www.example.com"],
-            "duration_seconds": 2,
-            "crawler_config": {"word_count_threshold": 10}
-        }
-        start_response = await client.post("/monitoring/profile/start", json=request_data)
-        session_id = start_response.json()["session_id"]
-        
-        # Get session immediately (should be running)
-        response = await client.get(f"/monitoring/profile/{session_id}")
-        assert response.status_code == 200
-        data = response.json()
-        
-        assert data["session_id"] == session_id
-        assert data["status"] in ["running", "completed"]
-        assert "started_at" in data
-        assert "urls" in data
-
-    @pytest.mark.asyncio
-    async def test_profiling_session_completion(self, client: AsyncClient):
-        """Test profiling session completes and produces results."""
-        # Start a short session
-        request_data = {
-            "urls": ["https://www.example.com"],
-            "duration_seconds": 3,
-            "crawler_config": {"word_count_threshold": 10}
-        }
-        start_response = await client.post("/monitoring/profile/start", json=request_data)
-        session_id = start_response.json()["session_id"]
-        
-        # Wait for completion
-        await asyncio.sleep(5)
-        
-        # Get completed session
-        response = await client.get(f"/monitoring/profile/{session_id}")
-        assert response.status_code == 200
-        data = response.json()
-        
-        assert data["status"] == "completed"
-        assert "completed_at" in data
-        assert "duration_seconds" in data
-        assert "results" in data
-        
-        # Verify results structure
-        results = data["results"]
-        assert "total_requests" in results
-        assert "successful_requests" in results
-        assert "failed_requests" in results
-        assert "avg_response_time_ms" in results
-        assert "system_metrics" in results
-
-    @pytest.mark.asyncio
-    async def test_profiling_session_not_found(self, client: AsyncClient):
-        """Test retrieving non-existent session returns 404."""
-        response = await client.get("/monitoring/profile/nonexistent-id-12345")
-        assert response.status_code == 404
-        data = response.json()
-        assert "detail" in data
-
-    @pytest.mark.asyncio
-    async def test_delete_profiling_session(self, client: AsyncClient):
-        """Test deleting a profiling session."""
-        # Start a session
-        request_data = {
-            "urls": ["https://www.example.com"],
-            "duration_seconds": 1,
-            "crawler_config": {"word_count_threshold": 10}
-        }
-        start_response = await client.post("/monitoring/profile/start", json=request_data)
-        session_id = start_response.json()["session_id"]
-        
-        # Wait for completion
-        await asyncio.sleep(2)
-        
-        # Delete session
-        delete_response = await client.delete(f"/monitoring/profile/{session_id}")
-        assert delete_response.status_code == 200
-        data = delete_response.json()
-        assert data["status"] == "deleted"
-        assert data["session_id"] == session_id
-        
-        # Verify it's gone
-        get_response = await client.get(f"/monitoring/profile/{session_id}")
-        assert get_response.status_code == 404
-
-    @pytest.mark.asyncio
-    async def test_cleanup_old_sessions(self, client: AsyncClient):
-        """Test cleaning up old profiling sessions."""
-        # Start a few sessions
-        for i in range(3):
-            request_data = {
-                "urls": ["https://www.example.com"],
-                "duration_seconds": 1,
-                "crawler_config": {"word_count_threshold": 10}
-            }
-            await client.post("/monitoring/profile/start", json=request_data)
-        
-        # Wait for completion
-        await asyncio.sleep(2)
-        
-        # Cleanup sessions older than 0 seconds (all completed ones)
-        cleanup_response = await client.post(
-            "/monitoring/profile/cleanup",
-            json={"max_age_seconds": 0}
-        )
-        assert cleanup_response.status_code == 200
-        data = cleanup_response.json()
-        assert "deleted_count" in data
-        assert data["deleted_count"] >= 0
-
-    @pytest.mark.asyncio
-    async def test_list_sessions_after_operations(self, client: AsyncClient):
-        """Test listing sessions shows correct state after various operations."""
-        # Start a session
-        request_data = {
-            "urls": ["https://www.example.com"],
-            "duration_seconds": 5,
-            "crawler_config": {"word_count_threshold": 10}
-        }
-        start_response = await client.post("/monitoring/profile/start", json=request_data)
-        session_id = start_response.json()["session_id"]
-        
-        # List sessions
-        list_response = await client.get("/monitoring/profile")
-        assert list_response.status_code == 200
-        data = list_response.json()
-        
-        # Should have at least one session
-        sessions = data["sessions"]
-        assert len(sessions) >= 1
-        
-        # Find our session
-        our_session = next((s for s in sessions if s["session_id"] == session_id), None)
-        assert our_session is not None
-        assert our_session["status"] in ["running", "completed"]
-
-
-class TestProfilingWithCrawlConfig:
-    """Tests for profiling with various crawler configurations."""
-
-    @pytest.mark.asyncio
-    async def test_profiling_with_extraction_strategy(self, client: AsyncClient):
-        """Test profiling with extraction strategy configured."""
-        request_data = {
-            "urls": ["https://www.example.com"],
-            "duration_seconds": 2,
-            "crawler_config": {
-                "word_count_threshold": 10,
-                "extraction_strategy": "NoExtractionStrategy"
-            }
-        }
-        
-        response = await client.post("/monitoring/profile/start", json=request_data)
-        assert response.status_code == 200
-        data = response.json()
-        assert data["status"] == "running"
-
-    @pytest.mark.asyncio
-    async def test_profiling_with_browser_config(self, client: AsyncClient):
-        """Test profiling with custom browser configuration."""
-        request_data = {
-            "urls": ["https://www.example.com"],
-            "duration_seconds": 2,
-            "browser_config": {
-                "headless": True,
-                "verbose": False
-            },
-            "crawler_config": {
-                "word_count_threshold": 10
-            }
-        }
-        
-        response = await client.post("/monitoring/profile/start", json=request_data)
-        assert response.status_code == 200
-        data = response.json()
-        assert data["status"] == "running"
-
-
-class TestIntegrationScenarios:
-    """Integration tests for real-world monitoring scenarios."""
-
-    @pytest.mark.asyncio
-    async def test_concurrent_crawls_and_monitoring(self, client: AsyncClient):
-        """Test monitoring multiple concurrent crawls."""
-        # Reset stats
-        await client.post("/monitoring/stats/reset")
-        
-        # Start multiple crawls concurrently
-        crawl_tasks = []
-        urls = [
-            "https://www.example.com",
-            "https://www.python.org",
-            "https://www.github.com"
-        ]
-        
-        for url in urls:
-            crawl_request = {
-                "urls": [url],
-                "crawler_config": {"word_count_threshold": 10}
-            }
-            task = client.post("/crawl", json=crawl_request)
-            crawl_tasks.append(task)
-        
-        # Execute concurrently
-        responses = await asyncio.gather(*crawl_tasks, return_exceptions=True)
-        
-        # Get stats
-        await asyncio.sleep(1)  # Give tracking time to update
-        stats_response = await client.get("/monitoring/stats")
-        stats = stats_response.json()
-        
-        # Should have tracked multiple crawls
-        assert stats["total_crawls"] >= len(urls)
-
-    @pytest.mark.asyncio
-    async def test_profiling_and_stats_correlation(self, client: AsyncClient):
-        """Test that profiling data correlates with statistics."""
-        # Reset stats
-        await client.post("/monitoring/stats/reset")
-        
-        # Start profiling session
-        profile_request = {
-            "urls": ["https://www.example.com"],
-            "duration_seconds": 3,
-            "crawler_config": {"word_count_threshold": 10}
-        }
-        profile_response = await client.post("/monitoring/profile/start", json=profile_request)
-        session_id = profile_response.json()["session_id"]
-        
-        # Wait for completion
-        await asyncio.sleep(5)
-        
-        # Get profiling results
-        profile_data_response = await client.get(f"/monitoring/profile/{session_id}")
-        profile_data = profile_data_response.json()
-        
-        # Get stats
-        stats_response = await client.get("/monitoring/stats")
-        stats = stats_response.json()
-        
-        # Stats should reflect profiling activity
-        assert stats["total_crawls"] >= profile_data["results"]["total_requests"]
-
-
-if __name__ == "__main__":
-    pytest.main([__file__, "-v", "-s"])

tests/docker/test_server_requests.py

@@ -34,9 +34,9 @@ from crawl4ai import (
 
 # --- Test Configuration ---
 # BASE_URL = os.getenv("CRAWL4AI_TEST_URL", "http://localhost:8020") # Make base URL configurable
-BASE_URL = os.getenv("CRAWL4AI_TEST_URL", "http://0.0.0.0:11234") # Make base URL configurable
+BASE_URL = os.getenv("CRAWL4AI_TEST_URL", "http://localhost:11235") # Make base URL configurable
 # Use a known simple HTML page for basic tests
-SIMPLE_HTML_URL = "https://docs.crawl4ai.com"
+SIMPLE_HTML_URL = "https://httpbin.org/html"
 # Use a site suitable for scraping tests
 SCRAPE_TARGET_URL = "http://books.toscrape.com/"
 # Use a site with internal links for deep crawl tests
@@ -78,37 +78,21 @@ async def process_streaming_response(response: httpx.Response) -> List[Dict[str,
     """Processes an NDJSON streaming response."""
     results = []
     completed = False
-    buffer = ""
-
-    async for chunk in response.aiter_text():
-        buffer += chunk
-        lines = buffer.split('\n')
-
-        # Keep the last incomplete line in buffer
-        buffer = lines.pop() if lines and not lines[-1].endswith('\n') else ""
-
-        for line in lines:
-            line = line.strip()
-            if not line:
-                continue
-
+    async for line in response.aiter_lines():
+        if line:
             try:
                 data = json.loads(line)
-                if data.get("status") in ["completed", "error"]:
+                if data.get("status") == "completed":
                     completed = True
-                    print(f"DEBUG: Received completion marker: {data}")  # Debug output
-                    break
+                    break # Stop processing after completion marker
                 else:
                     results.append(data)
             except json.JSONDecodeError:
                 pytest.fail(f"Failed to decode JSON line: {line}")
-
-        if completed:
-            break
-
-    print(f"DEBUG: Final results count: {len(results)}, completed: {completed}")  # Debug output
     assert completed, "Streaming response did not end with a completion marker."
     return results
+
+
 # --- Test Class ---
 
 @pytest.mark.asyncio
@@ -156,7 +140,7 @@ class TestCrawlEndpoints:
         await assert_crawl_result_structure(result)
         assert result["success"] is True
         assert result["url"] == SIMPLE_HTML_URL
-        assert "Crawl4AI Documentation" in result["html"]
+        assert "<h1>Herman Melville - Moby-Dick</h1>" in result["html"]
         # We don't specify a markdown generator in this test, so don't make assumptions about markdown field
         # It might be null, missing, or populated depending on the server's default behavior
     async def test_crawl_with_stream_direct(self, async_client: httpx.AsyncClient):
@@ -192,7 +176,7 @@ class TestCrawlEndpoints:
             await assert_crawl_result_structure(result)
             assert result["success"] is True
             assert result["url"] == SIMPLE_HTML_URL
-            assert "Crawl4AI Documentation" in result["html"]
+            assert "<h1>Herman Melville - Moby-Dick</h1>" in result["html"]
     async def test_simple_crawl_single_url_streaming(self, async_client: httpx.AsyncClient):
         """Test /crawl/stream with a single URL and simple config values."""
         payload = {
@@ -221,13 +205,13 @@ class TestCrawlEndpoints:
         await assert_crawl_result_structure(result)
         assert result["success"] is True
         assert result["url"] == SIMPLE_HTML_URL
-        assert "Crawl4AI Documentation" in result["html"]
+        assert "<h1>Herman Melville - Moby-Dick</h1>" in result["html"]
 
 
     # 2. Multi-URL and Dispatcher
     async def test_multi_url_crawl(self, async_client: httpx.AsyncClient):
         """Test /crawl with multiple URLs, implicitly testing dispatcher."""
-        urls = [SIMPLE_HTML_URL, "https://www.geeksforgeeks.org/"]
+        urls = [SIMPLE_HTML_URL, "https://httpbin.org/links/10/0"]
         payload = {
             "urls": urls,
             "browser_config": {
@@ -270,9 +254,8 @@ class TestCrawlEndpoints:
             assert result["url"] in urls
 
     async def test_multi_url_crawl_streaming(self, async_client: httpx.AsyncClient):
-
         """Test /crawl/stream with multiple URLs."""
-        urls = [SIMPLE_HTML_URL, "https://www.geeksforgeeks.org/"]
+        urls = [SIMPLE_HTML_URL, "https://httpbin.org/links/10/0"]
         payload = {
             "urls": urls,
             "browser_config": {
@@ -354,7 +337,7 @@ class TestCrawlEndpoints:
         assert isinstance(result["markdown"], dict)
         assert "raw_markdown" in result["markdown"]
         assert "fit_markdown" in result["markdown"] # Pruning creates fit_markdown
-        assert "Crawl4AI" in result["markdown"]["raw_markdown"]
+        assert "Moby-Dick" in result["markdown"]["raw_markdown"]
         # Fit markdown content might be different/shorter due to pruning
         assert len(result["markdown"]["fit_markdown"]) <= len(result["markdown"]["raw_markdown"])
 
@@ -605,9 +588,6 @@ class TestCrawlEndpoints:
               configured via .llm.env or environment variables.
               This test uses the default provider configured in the server's config.yml.
         """
-        # Skip test if no OpenAI API key is configured
-        if not os.getenv("OPENAI_API_KEY"):
-            pytest.skip("OPENAI_API_KEY not configured, skipping LLM extraction test")
         payload = {
             "urls": [SIMPLE_HTML_URL],
             "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
@@ -618,27 +598,26 @@ class TestCrawlEndpoints:
                     "extraction_strategy": {
                         "type": "LLMExtractionStrategy",
                         "params": {
-                            "instruction": "Extract the main title and any key information about Crawl4AI from the text into JSON.",
+                            "instruction": "Extract the main title and the author mentioned in the text into JSON.",
                             # LLMConfig is implicitly defined by server's config.yml and .llm.env
                             # If you needed to override provider/token PER REQUEST:
                             "llm_config": {
                                "type": "LLMConfig",
                                "params": {
-                                  "provider": "deepseek/deepseek-chat-v3.1:free", # Use deepseek model from openrouter
-                                  "api_token": os.getenv("OPENAI_API_KEY"), # Use OPENAI_API_KEY for openrouter
-                                  "base_url": "https://openrouter.ai/api/v1" # OpenRouter base URL
+                                  "provider": "openai/gpt-4o", # Example override
+                                  "api_token": os.getenv("OPENAI_API_KEY") # Example override
                                }
                             },
                             "schema": { # Optional: Provide a schema for structured output
                                 "type": "dict", # IMPORTANT: Wrap schema dict
                                 "value": {
-                                    "title": "Crawl4AI Info",
+                                    "title": "Book Info",
                                     "type": "object",
                                     "properties": {
-                                        "title": {"type": "string", "description": "The main title of the page"},
-                                        "description": {"type": "string", "description": "Key information about Crawl4AI"}
+                                        "title": {"type": "string", "description": "The main title of the work"},
+                                        "author": {"type": "string", "description": "The author of the work"}
                                     },
-                                     "required": ["title"]
+                                     "required": ["title", "author"]
                                 }
                             }
                         }
@@ -676,11 +655,15 @@ class TestCrawlEndpoints:
                 extracted_item = extracted_data[0]  # Take first item
                 assert isinstance(extracted_item, dict)
                 assert "title" in extracted_item
-                assert "Crawl4AI" in extracted_item.get("title", "")
+                assert "author" in extracted_item
+                assert "Moby-Dick" in extracted_item.get("title", "")
+                assert "Herman Melville" in extracted_item.get("author", "")
             else:
                 assert isinstance(extracted_data, dict)
                 assert "title" in extracted_data
-                assert "Crawl4AI" in extracted_data.get("title", "")
+                assert "author" in extracted_data
+                assert "Moby-Dick" in extracted_data.get("title", "")
+                assert "Herman Melville" in extracted_data.get("author", "")
         except (json.JSONDecodeError, AssertionError) as e:
             pytest.fail(f"LLM extracted content parsing or validation failed: {e}\nContent: {result['extracted_content']}")
         except Exception as e: # Catch any other unexpected error
@@ -700,9 +683,9 @@ class TestCrawlEndpoints:
         # Should return 200 with failed results, not 500
         print(f"Status code: {response.status_code}")
         print(f"Response: {response.text}")
-        assert response.status_code == 200
+        assert response.status_code == 500
         data = response.json()
-        assert data["success"] is True  # Overall success, but individual results may fail
+        assert data["detail"].startswith("Crawl request failed:")
 
     async def test_mixed_success_failure_urls(self, async_client: httpx.AsyncClient):
         """Test handling of mixed success/failure URLs."""
@@ -871,102 +854,6 @@ class TestCrawlEndpoints:
         response = await async_client.post("/config/dump", json=nested_payload)
         assert response.status_code == 400
 
-    async def test_llm_job_with_chunking_strategy(self, async_client: httpx.AsyncClient):
-        """Test LLM job endpoint with chunking strategy."""
-        payload = {
-            "url": SIMPLE_HTML_URL,
-            "q": "Extract the main title and any headings from the content",
-            "chunking_strategy": {
-                "type": "RegexChunking",
-                "params": {
-                    "patterns": ["\\n\\n+"],
-                    "overlap": 50
-                }
-            }
-        }
-        
-        try:
-            # Submit the job
-            response = await async_client.post("/llm/job", json=payload)
-            response.raise_for_status()
-            job_data = response.json()
-            
-            assert "task_id" in job_data
-            task_id = job_data["task_id"]
-            
-            # Poll for completion (simple implementation)
-            max_attempts = 10  # Reduced for testing
-            attempt = 0
-            while attempt < max_attempts:
-                status_response = await async_client.get(f"/llm/job/{task_id}")
-                
-                # Check if response is valid JSON
-                try:
-                    status_data = status_response.json()
-                except:
-                    print(f"Non-JSON response: {status_response.text}")
-                    attempt += 1
-                    await asyncio.sleep(1)
-                    continue
-                
-                if status_data.get("status") == "completed":
-                    # Verify we got a result
-                    assert "result" in status_data
-                    result = status_data["result"]
-                    # Result can be string, dict, or list depending on extraction
-                    assert result is not None
-                    print(f"✓ LLM job with chunking completed successfully. Result type: {type(result)}")
-                    break
-                elif status_data.get("status") == "failed":
-                    pytest.fail(f"LLM job failed: {status_data.get('error', 'Unknown error')}")
-                    break
-                else:
-                    attempt += 1
-                    await asyncio.sleep(1)  # Wait 1 second before checking again
-            
-            if attempt >= max_attempts:
-                # For testing purposes, just verify the job was submitted
-                print("✓ LLM job with chunking submitted successfully (completion check timed out)")
-                
-        except httpx.HTTPStatusError as e:
-            pytest.fail(f"LLM job request failed: {e}. Response: {e.response.text}")
-        except Exception as e:
-            pytest.fail(f"LLM job test failed: {e}")
-
-    async def test_chunking_strategies_supported(self, async_client: httpx.AsyncClient):
-        """Test that all chunking strategies are supported by the API."""
-        from deploy.docker.utils import create_chunking_strategy
-        
-        # Test all supported chunking strategies
-        strategies_to_test = [
-            {"type": "IdentityChunking", "params": {}},
-            {"type": "RegexChunking", "params": {"patterns": ["\\n\\n"]}},
-            {"type": "FixedLengthWordChunking", "params": {"chunk_size": 50}},
-            {"type": "SlidingWindowChunking", "params": {"window_size": 100, "step": 50}},
-            {"type": "OverlappingWindowChunking", "params": {"window_size": 100, "overlap": 20}},
-        ]
-        
-        for strategy_config in strategies_to_test:
-            try:
-                # Test that the strategy can be created
-                strategy = create_chunking_strategy(strategy_config)
-                assert strategy is not None
-                print(f"✓ {strategy_config['type']} strategy created successfully")
-                
-                # Test basic chunking functionality
-                test_text = "This is a test document with multiple sentences. It should be split appropriately."
-                chunks = strategy.chunk(test_text)
-                assert isinstance(chunks, list)
-                assert len(chunks) > 0
-                print(f"✓ {strategy_config['type']} chunking works: {len(chunks)} chunks")
-                
-            except Exception as e:
-                # Some strategies may fail due to missing dependencies (NLTK), but that's OK
-                if "NlpSentenceChunking" in strategy_config["type"] or "TopicSegmentationChunking" in strategy_config["type"]:
-                    print(f"⚠ {strategy_config['type']} requires NLTK dependencies: {e}")
-                else:
-                    pytest.fail(f"Unexpected error with {strategy_config['type']}: {e}")
-
     async def test_malformed_request_handling(self, async_client: httpx.AsyncClient):
         """Test handling of malformed requests."""
         # Test missing required fields
@@ -984,124 +871,6 @@ class TestCrawlEndpoints:
         response = await async_client.post("/crawl", json=empty_urls_payload)
         assert response.status_code == 422  # "At least one URL required"
 
-    # 7. HTTP-only Crawling Tests
-    async def test_http_crawl_single_url(self, async_client: httpx.AsyncClient):
-        """Test /crawl/http with a single URL using HTTP-only strategy."""
-        payload = {
-            "urls": [SIMPLE_HTML_URL],
-            "http_config": {
-                "method": "GET",
-                "headers": {"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},
-                "follow_redirects": True,
-                "verify_ssl": True
-            },
-            "crawler_config": {
-                "cache_mode": CacheMode.BYPASS.value,
-                "screenshot": False
-            }
-        }
-        try:
-            response = await async_client.post("/crawl/http", json=payload)
-            print(f"HTTP Response status: {response.status_code}")
-            response.raise_for_status()
-            data = response.json()
-        except httpx.HTTPStatusError as e:
-            print(f"HTTP Server error: {e}")
-            print(f"Response content: {e.response.text}")
-            raise
-
-        assert data["success"] is True
-        assert isinstance(data["results"], list)
-        assert len(data["results"]) == 1
-        result = data["results"][0]
-        await assert_crawl_result_structure(result)
-        assert result["success"] is True
-        assert result["url"] == SIMPLE_HTML_URL
-        assert "Crawl4AI Documentation" in result["html"]
-        # Check that processing was fast (HTTP should be much faster than browser)
-        assert data["server_processing_time_s"] < 5.0  # Should complete in under 5 seconds
-
-    async def test_http_crawl_streaming(self, async_client: httpx.AsyncClient):
-        """Test /crawl/http/stream with HTTP-only strategy."""
-        payload = {
-            "urls": [SIMPLE_HTML_URL],
-            "http_config": {
-                "method": "GET",
-                "headers": {"Accept": "text/html"},
-                "follow_redirects": True
-            },
-            "crawler_config": {
-                "cache_mode": CacheMode.BYPASS.value,
-                "screenshot": False
-            }
-        }
-        async with async_client.stream("POST", "/crawl/http/stream", json=payload) as response:
-            response.raise_for_status()
-            assert response.headers["content-type"] == "application/x-ndjson"
-            assert response.headers.get("x-stream-status") == "active"
-
-            results = await process_streaming_response(response)
-
-            assert len(results) == 1
-            result = results[0]
-            await assert_crawl_result_structure(result)
-            assert result["success"] is True
-            assert result["url"] == SIMPLE_HTML_URL
-            assert "Crawl4AI Documentation" in result["html"]
-
-    async def test_http_crawl_api_endpoint(self, async_client: httpx.AsyncClient):
-        """Test HTTP crawling with a JSON API endpoint."""
-        payload = {
-            "urls": ["https://httpbin.org/json"],
-            "http_config": {
-                "method": "GET",
-                "headers": {"Accept": "application/json"},
-                "follow_redirects": True
-            },
-            "crawler_config": {
-                "cache_mode": CacheMode.BYPASS.value
-            }
-        }
-        try:
-            response = await async_client.post("/crawl/http", json=payload)
-            response.raise_for_status()
-            data = response.json()
-        except httpx.HTTPStatusError as e:
-            print(f"HTTP API test error: {e}")
-            print(f"Response: {e.response.text}")
-            raise
-
-        assert data["success"] is True
-        assert len(data["results"]) == 1
-        result = data["results"][0]
-        assert result["success"] is True
-        assert result["url"] == "https://httpbin.org/json"
-        # Should contain JSON response
-        assert "slideshow" in result["html"] or "application/json" in result.get("content_type", "")
-
-    async def test_http_crawl_error_handling(self, async_client: httpx.AsyncClient):
-        """Test error handling for HTTP crawl endpoints."""
-        # Test invalid URL
-        invalid_payload = {
-            "urls": ["invalid-url"],
-            "http_config": {"method": "GET"},
-            "crawler_config": {"cache_mode": CacheMode.BYPASS.value}
-        }
-        response = await async_client.post("/crawl/http", json=invalid_payload)
-        # HTTP crawler handles invalid URLs gracefully, returns 200 with failed results
-        assert response.status_code == 200
-
-        # Test non-existent domain
-        nonexistent_payload = {
-            "urls": ["https://nonexistent-domain-12345.com"],
-            "http_config": {"method": "GET"},
-            "crawler_config": {"cache_mode": CacheMode.BYPASS.value}
-        }
-        response = await async_client.post("/crawl/http", json=nonexistent_payload)
-        # HTTP crawler handles unreachable hosts gracefully, returns 200 with failed results
-        assert response.status_code == 200
-
-
 if __name__ == "__main__":
     # Define arguments for pytest programmatically
     # -v: verbose output

tests/docker/test_table_extraction.py

@@ -1,458 +0,0 @@
-"""
-Integration tests for Table Extraction functionality in Crawl4AI Docker Server
-
-Tests cover:
-1. Integrated table extraction during crawls
-2. Dedicated /tables endpoints
-3. All extraction strategies (default, LLM, financial)
-4. Batch processing
-5. Error handling
-
-Note: These tests require the Docker server to be running on localhost:11235
-Run: python deploy/docker/server.py
-"""
-
-import pytest
-import requests
-import time
-from typing import Dict, Any
-
-
-# Base URL for the Docker API server
-BASE_URL = "http://localhost:11234"
-
-# Sample HTML with tables for testing
-SAMPLE_HTML_WITH_TABLES = """
-<!DOCTYPE html>
-<html>
-<head><title>Test Page with Tables</title></head>
-<body>
-    <h1>Financial Data</h1>
-    
-    <!-- Simple table -->
-    <table id="simple">
-        <tr><th>Name</th><th>Age</th></tr>
-        <tr><td>Alice</td><td>25</td></tr>
-        <tr><td>Bob</td><td>30</td></tr>
-    </table>
-    
-    <!-- Financial table -->
-    <table id="financial">
-        <thead>
-            <tr><th>Quarter</th><th>Revenue</th><th>Expenses</th><th>Profit</th></tr>
-        </thead>
-        <tbody>
-            <tr><td>Q1 2024</td><td>$1,250,000.00</td><td>$850,000.00</td><td>$400,000.00</td></tr>
-            <tr><td>Q2 2024</td><td>$1,500,000.00</td><td>$900,000.00</td><td>$600,000.00</td></tr>
-        </tbody>
-    </table>
-    
-    <!-- Complex nested table -->
-    <table id="complex">
-        <tr>
-            <th rowspan="2">Product</th>
-            <th colspan="2">Sales</th>
-        </tr>
-        <tr>
-            <th>Units</th>
-            <th>Revenue</th>
-        </tr>
-        <tr><td>Widget A</td><td>100</td><td>$5,000</td></tr>
-        <tr><td>Widget B</td><td>200</td><td>$10,000</td></tr>
-    </table>
-</body>
-</html>
-"""
-
-
-@pytest.fixture(scope="module")
-def server_url():
-    """Return the server URL"""
-    return BASE_URL
-
-
-@pytest.fixture(scope="module")
-def wait_for_server():
-    """Wait for server to be ready"""
-    max_retries = 5
-    for i in range(max_retries):
-        try:
-            response = requests.get(f"{BASE_URL}/health", timeout=2)
-            if response.status_code == 200:
-                return True
-        except requests.exceptions.RequestException:
-            if i < max_retries - 1:
-                time.sleep(1)
-    pytest.skip("Server not running on localhost:11235. Start with: python deploy/docker/server.py")
-
-
-class TestIntegratedTableExtraction:
-    """Test table extraction integrated with /crawl endpoint"""
-
-    def test_crawl_with_default_table_extraction(self, server_url, wait_for_server):
-        """Test crawling with default table extraction strategy"""
-        response = requests.post(f"{server_url}/crawl", json={
-            "urls": ["https://example.com/tables"],
-            "browser_config": {"headless": True},
-            "crawler_config": {},
-            "table_extraction": {
-                "strategy": "default"
-            }
-        })
-        
-        assert response.status_code == 200
-        data = response.json()
-        assert data["success"] is True
-        assert "results" in data
-        
-        # Check first result has tables
-        if data["results"]:
-            result = data["results"][0]
-            assert "tables" in result or result.get("success") is False
-
-    def test_crawl_with_llm_table_extraction(self, server_url, wait_for_server):
-        """Test crawling with LLM table extraction strategy"""
-        response = requests.post(f"{server_url}/crawl", json={
-            "urls": ["https://example.com/financial"],
-            "browser_config": {"headless": True},
-            "crawler_config": {},
-            "table_extraction": {
-                "strategy": "llm",
-                "llm_provider": "openai",
-                "llm_model": "gpt-4",
-                "llm_api_key": "test-key",
-                "llm_prompt": "Extract financial data from tables"
-            }
-        })
-        
-        # Should fail without valid API key, but structure should be correct
-        # In real scenario with valid key, this would succeed
-        assert response.status_code in [200, 500]  # May fail on auth
-
-    def test_crawl_with_financial_table_extraction(self, server_url, wait_for_server):
-        """Test crawling with financial table extraction strategy"""
-        response = requests.post(f"{server_url}/crawl", json={
-            "urls": ["https://example.com/stocks"],
-            "browser_config": {"headless": True},
-            "crawler_config": {},
-            "table_extraction": {
-                "strategy": "financial",
-                "preserve_formatting": True,
-                "extract_metadata": True
-            }
-        })
-        
-        assert response.status_code == 200
-        data = response.json()
-        assert data["success"] is True
-
-    def test_crawl_without_table_extraction(self, server_url, wait_for_server):
-        """Test crawling without table extraction (should work normally)"""
-        response = requests.post(f"{server_url}/crawl", json={
-            "urls": ["https://example.com"],
-            "browser_config": {"headless": True},
-            "crawler_config": {}
-        })
-        
-        assert response.status_code == 200
-        data = response.json()
-        assert data["success"] is True
-
-
-class TestDedicatedTableEndpoints:
-    """Test dedicated /tables endpoints"""
-
-    def test_extract_tables_from_html(self, server_url, wait_for_server):
-        """Test extracting tables from provided HTML"""
-        response = requests.post(f"{server_url}/tables/extract", json={
-            "html": SAMPLE_HTML_WITH_TABLES,
-            "config": {
-                "strategy": "default"
-            }
-        })
-        
-        assert response.status_code == 200
-        data = response.json()
-        assert data["success"] is True
-        assert data["table_count"] >= 3  # Should find at least 3 tables
-        assert "tables" in data
-        assert data["strategy"] == "default"
-        
-        # Verify table structure
-        if data["tables"]:
-            table = data["tables"][0]
-            assert "headers" in table or "rows" in table
-
-    def test_extract_tables_from_url(self, server_url, wait_for_server):
-        """Test extracting tables by fetching URL"""
-        response = requests.post(f"{server_url}/tables/extract", json={
-            "url": "https://example.com/tables",
-            "config": {
-                "strategy": "default"
-            }
-        })
-        
-        # May fail if URL doesn't exist, but structure should be correct
-        assert response.status_code in [200, 500]
-        
-        if response.status_code == 200:
-            data = response.json()
-            assert "success" in data
-            assert "tables" in data
-
-    def test_extract_tables_invalid_input(self, server_url, wait_for_server):
-        """Test error handling for invalid input"""
-        # No html or url provided
-        response = requests.post(f"{server_url}/tables/extract", json={
-            "config": {"strategy": "default"}
-        })
-        
-        assert response.status_code == 400
-        assert "html" in response.text.lower() or "url" in response.text.lower()
-
-    def test_extract_tables_both_html_and_url(self, server_url, wait_for_server):
-        """Test error when both html and url are provided"""
-        response = requests.post(f"{server_url}/tables/extract", json={
-            "html": "<table></table>",
-            "url": "https://example.com",
-            "config": {"strategy": "default"}
-        })
-        
-        assert response.status_code == 400
-        assert "both" in response.text.lower()
-
-
-class TestBatchTableExtraction:
-    """Test batch table extraction endpoints"""
-
-    def test_batch_extract_html_list(self, server_url, wait_for_server):
-        """Test batch extraction from multiple HTML contents"""
-        response = requests.post(f"{server_url}/tables/extract/batch", json={
-            "html_list": [
-                SAMPLE_HTML_WITH_TABLES,
-                "<table><tr><th>A</th></tr><tr><td>1</td></tr></table>",
-            ],
-            "config": {"strategy": "default"}
-        })
-        
-        assert response.status_code == 200
-        data = response.json()
-        assert data["success"] is True
-        assert "summary" in data
-        assert data["summary"]["total_processed"] == 2
-        assert data["summary"]["successful"] >= 0
-        assert "results" in data
-        assert len(data["results"]) == 2
-
-    def test_batch_extract_url_list(self, server_url, wait_for_server):
-        """Test batch extraction from multiple URLs"""
-        response = requests.post(f"{server_url}/tables/extract/batch", json={
-            "url_list": [
-                "https://example.com/page1",
-                "https://example.com/page2",
-            ],
-            "config": {"strategy": "default"}
-        })
-        
-        # May have mixed success/failure depending on URLs
-        assert response.status_code in [200, 500]
-        
-        if response.status_code == 200:
-            data = response.json()
-            assert "summary" in data
-            assert "results" in data
-
-    def test_batch_extract_mixed(self, server_url, wait_for_server):
-        """Test batch extraction from both HTML and URLs"""
-        response = requests.post(f"{server_url}/tables/extract/batch", json={
-            "html_list": [SAMPLE_HTML_WITH_TABLES],
-            "url_list": ["https://example.com/tables"],
-            "config": {"strategy": "default"}
-        })
-        
-        # May fail on URL crawling but should handle mixed input
-        assert response.status_code in [200, 500]
-        if response.status_code == 200:
-            data = response.json()
-            assert data["success"] is True
-            assert data["summary"]["total_processed"] == 2
-
-    def test_batch_extract_empty_list(self, server_url, wait_for_server):
-        """Test error when no items provided for batch"""
-        response = requests.post(f"{server_url}/tables/extract/batch", json={
-            "config": {"strategy": "default"}
-        })
-        
-        assert response.status_code == 400
-
-    def test_batch_extract_exceeds_limit(self, server_url, wait_for_server):
-        """Test error when batch size exceeds limit"""
-        response = requests.post(f"{server_url}/tables/extract/batch", json={
-            "html_list": ["<table></table>"] * 100,  # 100 items (limit is 50)
-            "config": {"strategy": "default"}
-        })
-        
-        assert response.status_code == 400
-        assert "50" in response.text or "limit" in response.text.lower()
-
-
-class TestTableExtractionStrategies:
-    """Test different table extraction strategies"""
-
-    def test_default_strategy(self, server_url, wait_for_server):
-        """Test default (regex-based) extraction strategy"""
-        response = requests.post(f"{server_url}/tables/extract", json={
-            "html": SAMPLE_HTML_WITH_TABLES,
-            "config": {
-                "strategy": "default"
-            }
-        })
-        
-        assert response.status_code == 200
-        data = response.json()
-        assert data["strategy"] == "default"
-        assert data["table_count"] >= 1
-
-    def test_llm_strategy_without_config(self, server_url, wait_for_server):
-        """Test LLM strategy without proper config (should use defaults or work)"""
-        response = requests.post(f"{server_url}/tables/extract", json={
-            "html": SAMPLE_HTML_WITH_TABLES,
-            "config": {
-                "strategy": "llm"
-                # Missing required LLM config
-            }
-        })
-        
-        # May succeed with defaults or fail - both are acceptable
-        assert response.status_code in [200, 400, 500]
-
-    def test_financial_strategy(self, server_url, wait_for_server):
-        """Test financial extraction strategy"""
-        response = requests.post(f"{server_url}/tables/extract", json={
-            "html": SAMPLE_HTML_WITH_TABLES,
-            "config": {
-                "strategy": "financial",
-                "preserve_formatting": True,
-                "extract_metadata": True
-            }
-        })
-        
-        assert response.status_code == 200
-        data = response.json()
-        assert data["strategy"] == "financial"
-        
-        # Financial tables should be extracted
-        if data["tables"]:
-            # Should find the financial table in our sample HTML
-            assert data["table_count"] >= 1
-
-    def test_none_strategy(self, server_url, wait_for_server):
-        """Test with 'none' strategy (no extraction)"""
-        response = requests.post(f"{server_url}/tables/extract", json={
-            "html": SAMPLE_HTML_WITH_TABLES,
-            "config": {
-                "strategy": "none"
-            }
-        })
-        
-        assert response.status_code == 200
-        data = response.json()
-        # Should return 0 tables
-        assert data["table_count"] == 0
-
-
-class TestTableExtractionConfig:
-    """Test table extraction configuration options"""
-
-    def test_preserve_formatting_option(self, server_url, wait_for_server):
-        """Test preserve_formatting option"""
-        response = requests.post(f"{server_url}/tables/extract", json={
-            "html": SAMPLE_HTML_WITH_TABLES,
-            "config": {
-                "strategy": "financial",
-                "preserve_formatting": True
-            }
-        })
-        
-        assert response.status_code == 200
-
-    def test_extract_metadata_option(self, server_url, wait_for_server):
-        """Test extract_metadata option"""
-        response = requests.post(f"{server_url}/tables/extract", json={
-            "html": SAMPLE_HTML_WITH_TABLES,
-            "config": {
-                "strategy": "financial",
-                "extract_metadata": True
-            }
-        })
-        
-        assert response.status_code == 200
-        data = response.json()
-        
-        # Check if tables have metadata when requested
-        if data["tables"]:
-            table = data["tables"][0]
-            assert isinstance(table, dict)
-
-
-class TestErrorHandling:
-    """Test error handling for table extraction"""
-
-    def test_malformed_html(self, server_url, wait_for_server):
-        """Test handling of malformed HTML"""
-        response = requests.post(f"{server_url}/tables/extract", json={
-            "html": "<table><tr><td>incomplete",
-            "config": {"strategy": "default"}
-        })
-        
-        # Should handle gracefully (either return empty or partial results)
-        assert response.status_code in [200, 400, 500]
-
-    def test_empty_html(self, server_url, wait_for_server):
-        """Test handling of empty HTML"""
-        response = requests.post(f"{server_url}/tables/extract", json={
-            "html": "",
-            "config": {"strategy": "default"}
-        })
-        
-        # May be rejected as invalid or processed as empty
-        assert response.status_code in [200, 400]
-        if response.status_code == 200:
-            data = response.json()
-            assert data["table_count"] == 0
-
-    def test_html_without_tables(self, server_url, wait_for_server):
-        """Test HTML with no tables"""
-        response = requests.post(f"{server_url}/tables/extract", json={
-            "html": "<html><body><p>No tables here</p></body></html>",
-            "config": {"strategy": "default"}
-        })
-        
-        assert response.status_code == 200
-        data = response.json()
-        assert data["table_count"] == 0
-
-    def test_invalid_strategy(self, server_url, wait_for_server):
-        """Test invalid strategy name"""
-        response = requests.post(f"{server_url}/tables/extract", json={
-            "html": SAMPLE_HTML_WITH_TABLES,
-            "config": {"strategy": "invalid_strategy"}
-        })
-        
-        # Should return validation error (400 or 422 from Pydantic)
-        assert response.status_code in [400, 422]
-
-    def test_missing_config(self, server_url, wait_for_server):
-        """Test missing configuration"""
-        response = requests.post(f"{server_url}/tables/extract", json={
-            "html": SAMPLE_HTML_WITH_TABLES
-            # Missing config
-        })
-        
-        # Should use default config or return error
-        assert response.status_code in [200, 400]
-
-
-# Run tests
-if __name__ == "__main__":
-    pytest.main([__file__, "-v"])

tests/docker/test_table_extraction_quick.py

@@ -1,225 +0,0 @@
-#!/usr/bin/env python3
-"""
-Quick test script for Table Extraction feature
-Tests the /tables/extract endpoint with sample HTML
-
-Usage:
-1. Start the server: python deploy/docker/server.py
-2. Run this script: python tests/docker/test_table_extraction_quick.py
-"""
-
-import requests
-import json
-import sys
-
-# Sample HTML with tables
-SAMPLE_HTML = """
-<!DOCTYPE html>
-<html>
-<body>
-    <h1>Test Tables</h1>
-    
-    <table id="simple">
-        <tr><th>Name</th><th>Age</th><th>City</th></tr>
-        <tr><td>Alice</td><td>25</td><td>New York</td></tr>
-        <tr><td>Bob</td><td>30</td><td>San Francisco</td></tr>
-        <tr><td>Charlie</td><td>35</td><td>Los Angeles</td></tr>
-    </table>
-    
-    <table id="financial">
-        <thead>
-            <tr><th>Quarter</th><th>Revenue</th><th>Profit</th></tr>
-        </thead>
-        <tbody>
-            <tr><td>Q1 2024</td><td>$1,250,000.00</td><td>$400,000.00</td></tr>
-            <tr><td>Q2 2024</td><td>$1,500,000.00</td><td>$600,000.00</td></tr>
-            <tr><td>Q3 2024</td><td>$1,750,000.00</td><td>$700,000.00</td></tr>
-        </tbody>
-    </table>
-</body>
-</html>
-"""
-
-BASE_URL = "http://localhost:11234"
-
-
-def test_server_health():
-    """Check if server is running"""
-    try:
-        response = requests.get(f"{BASE_URL}/health", timeout=2)
-        if response.status_code == 200:
-            print("✅ Server is running")
-            return True
-        else:
-            print(f"❌ Server health check failed: {response.status_code}")
-            return False
-    except requests.exceptions.RequestException as e:
-        print(f"❌ Server not reachable: {e}")
-        print("\n💡 Start the server with: python deploy/docker/server.py")
-        return False
-
-
-def test_default_strategy():
-    """Test default table extraction strategy"""
-    print("\n📊 Testing DEFAULT strategy...")
-    
-    response = requests.post(f"{BASE_URL}/tables/extract", json={
-        "html": SAMPLE_HTML,
-        "config": {
-            "strategy": "default"
-        }
-    })
-    
-    if response.status_code == 200:
-        data = response.json()
-        print(f"✅ Default strategy works!")
-        print(f"   - Table count: {data['table_count']}")
-        print(f"   - Strategy: {data['strategy']}")
-        
-        if data['tables']:
-            for idx, table in enumerate(data['tables']):
-                print(f"   - Table {idx + 1}: {len(table.get('rows', []))} rows")
-        
-        return True
-    else:
-        print(f"❌ Failed: {response.status_code}")
-        print(f"   Error: {response.text}")
-        return False
-
-
-def test_financial_strategy():
-    """Test financial table extraction strategy"""
-    print("\n💰 Testing FINANCIAL strategy...")
-    
-    response = requests.post(f"{BASE_URL}/tables/extract", json={
-        "html": SAMPLE_HTML,
-        "config": {
-            "strategy": "financial",
-            "preserve_formatting": True,
-            "extract_metadata": True
-        }
-    })
-    
-    if response.status_code == 200:
-        data = response.json()
-        print(f"✅ Financial strategy works!")
-        print(f"   - Table count: {data['table_count']}")
-        print(f"   - Strategy: {data['strategy']}")
-        return True
-    else:
-        print(f"❌ Failed: {response.status_code}")
-        print(f"   Error: {response.text}")
-        return False
-
-
-def test_none_strategy():
-    """Test none strategy (no extraction)"""
-    print("\n🚫 Testing NONE strategy...")
-    
-    response = requests.post(f"{BASE_URL}/tables/extract", json={
-        "html": SAMPLE_HTML,
-        "config": {
-            "strategy": "none"
-        }
-    })
-    
-    if response.status_code == 200:
-        data = response.json()
-        if data['table_count'] == 0:
-            print(f"✅ None strategy works (correctly extracted 0 tables)")
-            return True
-        else:
-            print(f"❌ None strategy returned {data['table_count']} tables (expected 0)")
-            return False
-    else:
-        print(f"❌ Failed: {response.status_code}")
-        return False
-
-
-def test_batch_extraction():
-    """Test batch extraction"""
-    print("\n📦 Testing BATCH extraction...")
-    
-    response = requests.post(f"{BASE_URL}/tables/extract/batch", json={
-        "html_list": [
-            SAMPLE_HTML,
-            "<table><tr><th>Col1</th></tr><tr><td>Val1</td></tr></table>"
-        ],
-        "config": {
-            "strategy": "default"
-        }
-    })
-    
-    if response.status_code == 200:
-        data = response.json()
-        print(f"✅ Batch extraction works!")
-        print(f"   - Total processed: {data['summary']['total_processed']}")
-        print(f"   - Successful: {data['summary']['successful']}")
-        print(f"   - Total tables: {data['summary']['total_tables_extracted']}")
-        return True
-    else:
-        print(f"❌ Failed: {response.status_code}")
-        print(f"   Error: {response.text}")
-        return False
-
-
-def test_error_handling():
-    """Test error handling"""
-    print("\n⚠️  Testing ERROR handling...")
-    
-    # Test with both html and url (should fail)
-    response = requests.post(f"{BASE_URL}/tables/extract", json={
-        "html": "<table></table>",
-        "url": "https://example.com",
-        "config": {"strategy": "default"}
-    })
-    
-    if response.status_code == 400:
-        print(f"✅ Error handling works (correctly rejected invalid input)")
-        return True
-    else:
-        print(f"❌ Expected 400 error, got: {response.status_code}")
-        return False
-
-
-def main():
-    print("=" * 60)
-    print("Table Extraction Feature - Quick Test")
-    print("=" * 60)
-    
-    # Check server
-    if not test_server_health():
-        sys.exit(1)
-    
-    # Run tests
-    results = []
-    results.append(("Default Strategy", test_default_strategy()))
-    results.append(("Financial Strategy", test_financial_strategy()))
-    results.append(("None Strategy", test_none_strategy()))
-    results.append(("Batch Extraction", test_batch_extraction()))
-    results.append(("Error Handling", test_error_handling()))
-    
-    # Summary
-    print("\n" + "=" * 60)
-    print("Test Summary")
-    print("=" * 60)
-    
-    passed = sum(1 for _, result in results if result)
-    total = len(results)
-    
-    for name, result in results:
-        status = "✅ PASS" if result else "❌ FAIL"
-        print(f"{status}: {name}")
-    
-    print(f"\nTotal: {passed}/{total} tests passed")
-    
-    if passed == total:
-        print("\n🎉 All tests passed! Table extraction is working correctly!")
-        sys.exit(0)
-    else:
-        print(f"\n⚠️  {total - passed} test(s) failed")
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

tests/example_url_discovery.py

@@ -1,239 +0,0 @@
-#!/usr/bin/env python3
-"""
-Runnable example for the /urls/discover endpoint.
-
-This script demonstrates how to use the new URL Discovery API endpoint
-to find relevant URLs from a domain before committing to a full crawl.
-"""
-
-import asyncio
-import httpx
-import json
-from typing import List, Dict, Any
-
-# Configuration
-BASE_URL = "http://localhost:11235"
-EXAMPLE_DOMAIN = "nbcnews.com"
-
-
-async def discover_urls_basic_example():
-    """Basic example of URL discovery."""
-    print("🔍 Basic URL Discovery Example")
-    print("=" * 50)
-    
-    # Basic discovery request
-    request_data = {
-        "domain": EXAMPLE_DOMAIN,
-        "seeding_config": {
-            "source": "sitemap",      # Use sitemap for fast discovery
-            "max_urls": 10           # Limit to 10 URLs
-        }
-    }
-    
-    async with httpx.AsyncClient() as client:
-        try:
-            response = await client.post(
-                f"{BASE_URL}/urls/discover",
-                json=request_data,
-                timeout=30.0
-            )
-            response.raise_for_status()
-            
-            urls = response.json()
-            print(f"✅ Found {len(urls)} URLs")
-            
-            # Display first few URLs
-            for i, url_obj in enumerate(urls[:3]):
-                print(f"  {i+1}. {url_obj.get('url', 'N/A')}")
-                
-            return urls
-            
-        except httpx.HTTPStatusError as e:
-            print(f"❌ HTTP Error: {e.response.status_code}")
-            print(f"Response: {e.response.text}")
-            return []
-        except Exception as e:
-            print(f"❌ Error: {e}")
-            return []
-
-
-async def discover_urls_advanced_example():
-    """Advanced example with filtering and metadata extraction."""
-    print("\n🎯 Advanced URL Discovery Example")
-    print("=" * 50)
-    
-    # Advanced discovery with filtering
-    request_data = {
-        "domain": EXAMPLE_DOMAIN,
-        "seeding_config": {
-            "source": "sitemap+cc",   # Use both sitemap and Common Crawl
-            "pattern": "*/news/*",    # Filter to news articles only
-            "extract_head": True,     # Extract page metadata
-            "max_urls": 5,
-            "live_check": True,       # Verify URLs are accessible
-            "verbose": True
-        }
-    }
-    
-    async with httpx.AsyncClient() as client:
-        try:
-            response = await client.post(
-                f"{BASE_URL}/urls/discover",
-                json=request_data,
-                timeout=60.0  # Longer timeout for advanced features
-            )
-            response.raise_for_status()
-            
-            urls = response.json()
-            print(f"✅ Found {len(urls)} news URLs with metadata")
-            
-            # Display URLs with metadata
-            for i, url_obj in enumerate(urls[:3]):
-                print(f"\n  {i+1}. URL: {url_obj.get('url', 'N/A')}")
-                print(f"     Status: {url_obj.get('status', 'unknown')}")
-                
-                head_data = url_obj.get('head_data', {})
-                if head_data:
-                    title = head_data.get('title', 'No title')
-                    description = head_data.get('description', 'No description')
-                    print(f"     Title: {title[:60]}...")
-                    print(f"     Description: {description[:60]}...")
-                
-            return urls
-            
-        except httpx.HTTPStatusError as e:
-            print(f"❌ HTTP Error: {e.response.status_code}")
-            print(f"Response: {e.response.text}")
-            return []
-        except Exception as e:
-            print(f"❌ Error: {e}")
-            return []
-
-
-async def discover_urls_with_scoring_example():
-    """Example using BM25 relevance scoring."""
-    print("\n🏆 URL Discovery with Relevance Scoring")
-    print("=" * 50)
-    
-    # Discovery with relevance scoring
-    request_data = {
-        "domain": EXAMPLE_DOMAIN,
-        "seeding_config": {
-            "source": "sitemap",
-            "extract_head": True,     # Required for BM25 scoring
-            "query": "politics election",  # Search for political content
-            "scoring_method": "bm25",
-            "score_threshold": 0.1,   # Minimum relevance score
-            "max_urls": 5
-        }
-    }
-    
-    async with httpx.AsyncClient() as client:
-        try:
-            response = await client.post(
-                f"{BASE_URL}/urls/discover",
-                json=request_data,
-                timeout=60.0
-            )
-            response.raise_for_status()
-            
-            urls = response.json()
-            print(f"✅ Found {len(urls)} relevant URLs")
-            
-            # Display URLs sorted by relevance score
-            for i, url_obj in enumerate(urls[:3]):
-                score = url_obj.get('score', 0)
-                print(f"\n  {i+1}. Score: {score:.3f}")
-                print(f"     URL: {url_obj.get('url', 'N/A')}")
-                
-                head_data = url_obj.get('head_data', {})
-                if head_data:
-                    title = head_data.get('title', 'No title')
-                    print(f"     Title: {title[:60]}...")
-                
-            return urls
-            
-        except httpx.HTTPStatusError as e:
-            print(f"❌ HTTP Error: {e.response.status_code}")
-            print(f"Response: {e.response.text}")
-            return []
-        except Exception as e:
-            print(f"❌ Error: {e}")
-            return []
-
-
-def demonstrate_request_schema():
-    """Show the complete request schema with all options."""
-    print("\n📋 Complete Request Schema")
-    print("=" * 50)
-    
-    complete_schema = {
-        "domain": "example.com",  # Required: Domain to discover URLs from
-        "seeding_config": {       # Optional: Configuration object
-            # Discovery sources
-            "source": "sitemap+cc",           # "sitemap", "cc", or "sitemap+cc"
-            
-            # Filtering options
-            "pattern": "*/blog/*",            # URL pattern filter (glob style)
-            "max_urls": 50,                   # Maximum URLs to return (-1 = no limit)
-            "filter_nonsense_urls": True,     # Filter out nonsense URLs
-            
-            # Metadata and validation
-            "extract_head": True,             # Extract <head> metadata
-            "live_check": True,               # Verify URL accessibility
-            
-            # Performance and rate limiting
-            "concurrency": 100,               # Concurrent requests
-            "hits_per_sec": 10,              # Rate limit (requests/second)
-            "force": False,                   # Bypass cache
-            
-            # Relevance scoring (requires extract_head=True)
-            "query": "search terms",          # Query for BM25 scoring
-            "scoring_method": "bm25",         # Scoring algorithm
-            "score_threshold": 0.2,           # Minimum score threshold
-            
-            # Debugging
-            "verbose": True                   # Enable verbose logging
-        }
-    }
-    
-    print("Full request schema:")
-    print(json.dumps(complete_schema, indent=2))
-    
-
-async def main():
-    """Run all examples."""
-    print("🚀 URL Discovery API Examples")
-    print("=" * 50)
-    print(f"Server: {BASE_URL}")
-    print(f"Domain: {EXAMPLE_DOMAIN}")
-    
-    # Check if server is running
-    async with httpx.AsyncClient() as client:
-        try:
-            response = await client.get(f"{BASE_URL}/health", timeout=5.0)
-            response.raise_for_status()
-            print("✅ Server is running\n")
-        except Exception as e:
-            print(f"❌ Server not available: {e}")
-            print("Please start the Crawl4AI server first:")
-            print("  docker compose up crawl4ai -d")
-            return
-    
-    # Run examples
-    await discover_urls_basic_example()
-    await discover_urls_advanced_example()
-    await discover_urls_with_scoring_example()
-    
-    # Show schema
-    demonstrate_request_schema()
-    
-    print("\n🎉 Examples complete!")
-    print("\nNext steps:")
-    print("1. Use discovered URLs with the /crawl endpoint")
-    print("2. Filter URLs based on your specific needs")
-    print("3. Combine with other API endpoints for complete workflows")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

tests/test_docker.py

@@ -70,7 +70,6 @@ def test_docker_deployment(version="basic"):
     # test_llm_extraction(tester)
     # test_llm_with_ollama(tester)
     # test_screenshot(tester)
-    test_link_analysis(tester)
 
 
 def test_basic_crawl(tester: Crawl4AiTester):
@@ -294,77 +293,6 @@ def test_screenshot(tester: Crawl4AiTester):
     assert result["result"]["success"]
 
 
-def test_link_analysis(tester: Crawl4AiTester):
-    print("\n=== Testing Link Analysis ===")
-
-    # Get auth token first
-    try:
-        token_response = requests.post(f"{tester.base_url}/token", json={"email": "test@example.com"})
-        token = token_response.json()["access_token"]
-        headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
-    except Exception as e:
-        print(f"Could not get auth token: {e}")
-        headers = {"Content-Type": "application/json"}
-
-    # Test basic link analysis
-    request_data = {
-        "url": "https://www.nbcnews.com/business"
-    }
-
-    response = requests.post(
-        f"{tester.base_url}/links/analyze",
-        headers=headers,
-        json=request_data,
-        timeout=60
-    )
-
-    if response.status_code == 200:
-        result = response.json()
-        total_links = sum(len(links) for links in result.values())
-        print(f"Link analysis successful: found {total_links} links")
-
-        # Check for expected categories
-        categories_found = []
-        for category in ['internal', 'external', 'social', 'download', 'email', 'phone']:
-            if category in result and result[category]:
-                categories_found.append(category)
-
-        print(f"Link categories found: {categories_found}")
-
-        # Verify we have some links
-        assert total_links > 0, "Should find at least one link"
-        assert len(categories_found) > 0, "Should find at least one link category"
-
-        # Test with configuration
-        request_data_with_config = {
-            "url": "https://www.nbcnews.com/business",
-            "config": {
-                "simulate_user": True,
-                "override_navigator": True,
-                "word_count_threshold": 1
-            }
-        }
-
-        response_with_config = requests.post(
-            f"{tester.base_url}/links/analyze",
-            headers=headers,
-            json=request_data_with_config,
-            timeout=60
-        )
-
-        if response_with_config.status_code == 200:
-            result_with_config = response_with_config.json()
-            total_links_config = sum(len(links) for links in result_with_config.values())
-            print(f"Link analysis with config: found {total_links_config} links")
-            assert total_links_config > 0, "Should find links even with config"
-
-        print("✅ Link analysis tests passed")
-    else:
-        print(f"❌ Link analysis failed: {response.status_code} - {response.text}")
-        # Don't fail the entire test suite for this endpoint
-        print("⚠️  Link analysis test failed, but continuing with other tests")
-
-
 if __name__ == "__main__":
     version = sys.argv[1] if len(sys.argv) > 1 else "basic"
     # version = "full"

tests/test_implementation.py

@@ -1,160 +0,0 @@
-#!/usr/bin/env python3
-"""
-Test script for the new URL discovery functionality.
-This tests the handler function directly without running the full server.
-"""
-
-import asyncio
-import sys
-import os
-from pathlib import Path
-
-# Add the repo to Python path
-repo_root = Path(__file__).parent
-sys.path.insert(0, str(repo_root))
-sys.path.insert(0, str(repo_root / "deploy" / "docker"))
-
-from rich.console import Console
-from rich.panel import Panel
-from rich.syntax import Syntax
-
-console = Console()
-
-async def test_url_discovery_handler():
-    """Test the URL discovery handler function directly."""
-    try:
-        # Import the handler function and dependencies
-        from api import handle_url_discovery
-        from crawl4ai.async_configs import SeedingConfig
-        
-        console.print("[bold cyan]Testing URL Discovery Handler Function[/bold cyan]")
-        
-        # Test 1: Basic functionality
-        console.print("\n[cyan]Test 1: Basic URL discovery[/cyan]")
-        
-        domain = "docs.crawl4ai.com"
-        seeding_config = {
-            "source": "sitemap",
-            "max_urls": 3,
-            "verbose": True
-        }
-        
-        console.print(f"[blue]Domain:[/blue] {domain}")
-        console.print(f"[blue]Config:[/blue] {seeding_config}")
-        
-        # Call the handler directly
-        result = await handle_url_discovery(domain, seeding_config)
-        
-        console.print(f"[green]✓ Handler executed successfully[/green]")
-        console.print(f"[green]✓ Result type: {type(result)}[/green]")
-        console.print(f"[green]✓ Result length: {len(result)}[/green]")
-        
-        # Print first few results if any
-        if result and len(result) > 0:
-            console.print("\n[blue]Sample results:[/blue]")
-            for i, url_obj in enumerate(result[:2]):
-                console.print(f"  {i+1}. {url_obj}")
-        
-        return True
-        
-    except ImportError as e:
-        console.print(f"[red]✗ Import error: {e}[/red]")
-        console.print("[yellow]This suggests missing dependencies or module structure issues[/yellow]")
-        return False
-    except Exception as e:
-        console.print(f"[red]✗ Handler error: {e}[/red]")
-        return False
-
-async def test_seeding_config_validation():
-    """Test SeedingConfig validation."""
-    try:
-        from crawl4ai.async_configs import SeedingConfig
-        
-        console.print("\n[cyan]Test 2: SeedingConfig validation[/cyan]")
-        
-        # Test valid config
-        valid_config = {
-            "source": "sitemap",
-            "max_urls": 5,
-            "pattern": "*"
-        }
-        
-        config = SeedingConfig(**valid_config)
-        console.print(f"[green]✓ Valid config created: {config.source}, max_urls={config.max_urls}[/green]")
-        
-        # Test invalid config
-        try:
-            invalid_config = {
-                "source": "invalid_source",
-                "max_urls": 5
-            }
-            config = SeedingConfig(**invalid_config)
-            console.print(f"[yellow]? Invalid config unexpectedly accepted[/yellow]")
-        except Exception as e:
-            console.print(f"[green]✓ Invalid config correctly rejected: {str(e)[:50]}...[/green]")
-        
-        return True
-        
-    except Exception as e:
-        console.print(f"[red]✗ SeedingConfig test error: {e}[/red]")
-        return False
-
-async def test_schema_validation():
-    """Test the URLDiscoveryRequest schema."""
-    try:
-        from schemas import URLDiscoveryRequest
-        
-        console.print("\n[cyan]Test 3: URLDiscoveryRequest schema validation[/cyan]")
-        
-        # Test valid request
-        valid_request_data = {
-            "domain": "example.com",
-            "seeding_config": {
-                "source": "sitemap",
-                "max_urls": 10
-            }
-        }
-        
-        request = URLDiscoveryRequest(**valid_request_data)
-        console.print(f"[green]✓ Valid request created: domain={request.domain}[/green]")
-        
-        # Test request with default config
-        minimal_request_data = {
-            "domain": "example.com"
-        }
-        
-        request = URLDiscoveryRequest(**minimal_request_data)
-        console.print(f"[green]✓ Minimal request created with defaults[/green]")
-        
-        return True
-        
-    except Exception as e:
-        console.print(f"[red]✗ Schema test error: {e}[/red]")
-        return False
-
-async def main():
-    """Run all tests."""
-    console.print("[bold blue]🔍 URL Discovery Implementation Tests[/bold blue]")
-    
-    results = []
-    
-    # Test the implementation components
-    results.append(await test_seeding_config_validation())
-    results.append(await test_schema_validation())
-    results.append(await test_url_discovery_handler())
-    
-    # Summary
-    console.print("\n[bold cyan]Test Summary[/bold cyan]")
-    passed = sum(results)
-    total = len(results)
-    
-    if passed == total:
-        console.print(f"[bold green]✓ All {total} implementation tests passed![/bold green]")
-        console.print("[green]The URL discovery endpoint is ready for integration testing[/green]")
-    else:
-        console.print(f"[bold yellow]⚠ {passed}/{total} tests passed[/bold yellow]")
-    
-    return passed == total
-
-if __name__ == "__main__":
-    asyncio.run(main())

tests/test_link_analysis.py

@@ -1,759 +0,0 @@
-import requests
-import json
-import time
-import sys
-import os
-from typing import Dict, Any, List
-
-
-class LinkAnalysisTester:
-    def __init__(self, base_url: str = "http://localhost:11234"):
-        self.base_url = base_url
-        self.token = self.get_test_token()
-
-    def get_test_token(self) -> str:
-        """Get authentication token for testing"""
-        try:
-            # Try to get token using test email
-            response = requests.post(
-                f"{self.base_url}/token",
-                json={"email": "test@example.com"},
-                timeout=10
-            )
-            if response.status_code == 200:
-                return response.json()["access_token"]
-        except Exception:
-            pass
-
-        # Fallback: try with common test token or skip auth for local testing
-        return "test-token"
-
-    def analyze_links(
-        self,
-        url: str,
-        config: Dict[str, Any] = None,
-        timeout: int = 60
-    ) -> Dict[str, Any]:
-        """Analyze links on a webpage"""
-        headers = {
-            "Content-Type": "application/json"
-        }
-
-        # Add auth if token is available
-        if self.token and self.token != "test-token":
-            headers["Authorization"] = f"Bearer {self.token}"
-
-        request_data = {"url": url}
-        if config:
-            request_data["config"] = config
-
-        response = requests.post(
-            f"{self.base_url}/links/analyze",
-            headers=headers,
-            json=request_data,
-            timeout=timeout
-        )
-
-        if response.status_code != 200:
-            raise Exception(f"Link analysis failed: {response.status_code} - {response.text}")
-
-        return response.json()
-
-
-def test_link_analysis_basic():
-    """Test basic link analysis functionality"""
-    print("\n=== Testing Basic Link Analysis ===")
-
-    tester = LinkAnalysisTester()
-
-    # Test with a simple page
-    test_url = "https://httpbin.org/links/10"
-
-    try:
-        result = tester.analyze_links(test_url)
-        print(f"✅ Successfully analyzed links on {test_url}")
-
-        # Check response structure
-        expected_categories = ['internal', 'external', 'social', 'download', 'email', 'phone']
-        found_categories = [cat for cat in expected_categories if cat in result]
-
-        print(f"📊 Found link categories: {found_categories}")
-
-        # Count total links
-        total_links = sum(len(links) for links in result.values())
-        print(f"🔗 Total links found: {total_links}")
-
-        # Verify link objects have expected fields
-        for category, links in result.items():
-            if links and len(links) > 0:
-                sample_link = links[0]
-                expected_fields = ['href', 'text']
-                optional_fields = ['title', 'base_domain', 'intrinsic_score', 'contextual_score', 'total_score']
-
-                missing_required = [field for field in expected_fields if field not in sample_link]
-                found_optional = [field for field in optional_fields if field in sample_link]
-
-                if missing_required:
-                    print(f"⚠️  Missing required fields in {category}: {missing_required}")
-                else:
-                    print(f"✅ {category} links have proper structure (has {len(found_optional)} optional fields: {found_optional})")
-
-        assert total_links > 0, "Should find at least one link"
-        print("✅ Basic link analysis test passed")
-
-    except Exception as e:
-        print(f"❌ Basic link analysis test failed: {str(e)}")
-        raise
-
-
-def test_link_analysis_with_config():
-    """Test link analysis with custom configuration"""
-    print("\n=== Testing Link Analysis with Config ===")
-
-    tester = LinkAnalysisTester()
-
-    # Test with valid LinkPreviewConfig options
-    config = {
-        "include_internal": True,
-        "include_external": True,
-        "max_links": 50,
-        "score_threshold": 0.3,
-        "verbose": True
-    }
-
-    test_url = "https://httpbin.org/links/10"
-
-    try:
-        result = tester.analyze_links(test_url, config)
-        print(f"✅ Successfully analyzed links with custom config")
-
-        # Verify configuration was applied
-        total_links = sum(len(links) for links in result.values())
-        print(f"🔗 Links found with config: {total_links}")
-
-        assert total_links > 0, "Should find links even with config"
-        print("✅ Config test passed")
-
-    except Exception as e:
-        print(f"❌ Config test failed: {str(e)}")
-        raise
-
-
-def test_link_analysis_complex_page():
-    """Test link analysis on a more complex page"""
-    print("\n=== Testing Link Analysis on Complex Page ===")
-
-    tester = LinkAnalysisTester()
-
-    # Test with a real-world page
-    test_url = "https://www.python.org"
-
-    try:
-        result = tester.analyze_links(test_url)
-        print(f"✅ Successfully analyzed links on {test_url}")
-
-        # Analyze link distribution
-        category_counts = {}
-        for category, links in result.items():
-            if links:
-                category_counts[category] = len(links)
-                print(f"📂 {category}: {len(links)} links")
-
-        # Find top-scoring links
-        all_links = []
-        for category, links in result.items():
-            if links:
-                for link in links:
-                    link['category'] = category
-                    all_links.append(link)
-
-        if all_links:
-            # Use intrinsic_score or total_score if available, fallback to 0
-            top_links = sorted(all_links, key=lambda x: x.get('total_score', x.get('intrinsic_score', 0)), reverse=True)[:5]
-            print("\n🏆 Top 5 links by score:")
-            for i, link in enumerate(top_links, 1):
-                score = link.get('total_score', link.get('intrinsic_score', 0))
-                print(f"  {i}. {link.get('text', 'N/A')} ({score:.2f}) - {link.get('category', 'unknown')}")
-
-        # Verify we found different types of links
-        assert len(category_counts) > 0, "Should find at least one link category"
-        print("✅ Complex page analysis test passed")
-
-    except Exception as e:
-        print(f"❌ Complex page analysis test failed: {str(e)}")
-        # Don't fail the test suite for network issues
-        print("⚠️  This test may fail due to network connectivity issues")
-
-
-def test_link_analysis_scoring():
-    """Test link scoring functionality"""
-    print("\n=== Testing Link Scoring ===")
-
-    tester = LinkAnalysisTester()
-
-    test_url = "https://httpbin.org/links/10"
-
-    try:
-        result = tester.analyze_links(test_url)
-
-        # Analyze score distribution
-        all_scores = []
-        for category, links in result.items():
-            if links:
-                for link in links:
-                    # Use total_score or intrinsic_score if available
-                    score = link.get('total_score', link.get('intrinsic_score', 0))
-                    if score is not None:  # Only include links that have scores
-                        all_scores.append(score)
-
-        if all_scores:
-            avg_score = sum(all_scores) / len(all_scores)
-            max_score = max(all_scores)
-            min_score = min(all_scores)
-
-            print(f"📊 Score statistics:")
-            print(f"   Average: {avg_score:.3f}")
-            print(f"   Maximum: {max_score:.3f}")
-            print(f"   Minimum: {min_score:.3f}")
-            print(f"   Total links scored: {len(all_scores)}")
-
-            # Verify scores are in expected range
-            assert all(0 <= score <= 1 for score in all_scores), "Scores should be between 0 and 1"
-            print("✅ All scores are in valid range")
-
-        print("✅ Link scoring test passed")
-
-    except Exception as e:
-        print(f"❌ Link scoring test failed: {str(e)}")
-        raise
-
-
-def test_link_analysis_error_handling():
-    """Test error handling for invalid requests"""
-    print("\n=== Testing Error Handling ===")
-
-    tester = LinkAnalysisTester()
-
-    # Test with invalid URL
-    try:
-        tester.analyze_links("not-a-valid-url")
-        print("⚠️  Expected error for invalid URL, but got success")
-    except Exception as e:
-        print(f"✅ Correctly handled invalid URL: {str(e)}")
-
-    # Test with non-existent URL
-    try:
-        result = tester.analyze_links("https://this-domain-does-not-exist-12345.com")
-        print("⚠️  This should have failed for non-existent domain")
-    except Exception as e:
-        print(f"✅ Correctly handled non-existent domain: {str(e)}")
-
-    print("✅ Error handling test passed")
-
-
-def test_link_analysis_performance():
-    """Test performance of link analysis"""
-    print("\n=== Testing Performance ===")
-
-    tester = LinkAnalysisTester()
-
-    test_url = "https://httpbin.org/links/50"
-
-    try:
-        start_time = time.time()
-        result = tester.analyze_links(test_url)
-        end_time = time.time()
-
-        duration = end_time - start_time
-        total_links = sum(len(links) for links in result.values())
-
-        print(f"⏱️  Analysis completed in {duration:.2f} seconds")
-        print(f"🔗 Found {total_links} links")
-        print(f"📈 Rate: {total_links/duration:.1f} links/second")
-
-        # Performance should be reasonable
-        assert duration < 60, f"Analysis took too long: {duration:.2f}s"
-        print("✅ Performance test passed")
-
-    except Exception as e:
-        print(f"❌ Performance test failed: {str(e)}")
-        raise
-
-
-def test_link_analysis_categorization():
-    """Test link categorization functionality"""
-    print("\n=== Testing Link Categorization ===")
-
-    tester = LinkAnalysisTester()
-
-    test_url = "https://www.python.org"
-
-    try:
-        result = tester.analyze_links(test_url)
-
-        # Check categorization
-        categories_found = []
-        for category, links in result.items():
-            if links:
-                categories_found.append(category)
-                print(f"📂 {category}: {len(links)} links")
-
-                # Analyze a sample link from each category
-                sample_link = links[0]
-                url = sample_link.get('href', '')
-                text = sample_link.get('text', '')
-                score = sample_link.get('total_score', sample_link.get('intrinsic_score', 0))
-
-                print(f"   Sample: {text[:50]}... ({url[:50]}...) - score: {score:.2f}")
-
-        print(f"✅ Found {len(categories_found)} link categories")
-        print("✅ Categorization test passed")
-
-    except Exception as e:
-        print(f"❌ Categorization test failed: {str(e)}")
-        # Don't fail for network issues
-        print("⚠️  This test may fail due to network connectivity issues")
-
-
-def test_link_analysis_all_config_options():
-    """Test all available LinkPreviewConfig options"""
-    print("\n=== Testing All Configuration Options ===")
-
-    tester = LinkAnalysisTester()
-    test_url = "https://httpbin.org/links/10"
-
-    # Test 1: include_internal and include_external
-    print("\n🔍 Testing include_internal/include_external options...")
-
-    configs = [
-        {
-            "name": "Internal only",
-            "config": {"include_internal": True, "include_external": False}
-        },
-        {
-            "name": "External only",
-            "config": {"include_internal": False, "include_external": True}
-        },
-        {
-            "name": "Both internal and external",
-            "config": {"include_internal": True, "include_external": True}
-        }
-    ]
-
-    for test_case in configs:
-        try:
-            result = tester.analyze_links(test_url, test_case["config"])
-            internal_count = len(result.get('internal', []))
-            external_count = len(result.get('external', []))
-
-            print(f"   {test_case['name']}: {internal_count} internal, {external_count} external links")
-
-            # Verify configuration behavior
-            if test_case["config"]["include_internal"] and not test_case["config"]["include_external"]:
-                assert internal_count >= 0, "Should have internal links"
-            elif not test_case["config"]["include_internal"] and test_case["config"]["include_external"]:
-                assert external_count >= 0, "Should have external links"
-
-        except Exception as e:
-            print(f"   ❌ {test_case['name']} failed: {e}")
-
-    # Test 2: include_patterns and exclude_patterns
-    print("\n🔍 Testing include/exclude patterns...")
-
-    pattern_configs = [
-        {
-            "name": "Include specific patterns",
-            "config": {
-                "include_patterns": ["*/links/*", "*/test*"],
-                "include_internal": True,
-                "include_external": True
-            }
-        },
-        {
-            "name": "Exclude specific patterns",
-            "config": {
-                "exclude_patterns": ["*/admin*", "*/login*"],
-                "include_internal": True,
-                "include_external": True
-            }
-        },
-        {
-            "name": "Both include and exclude patterns",
-            "config": {
-                "include_patterns": ["*"],
-                "exclude_patterns": ["*/exclude*"],
-                "include_internal": True,
-                "include_external": True
-            }
-        }
-    ]
-
-    for test_case in pattern_configs:
-        try:
-            result = tester.analyze_links(test_url, test_case["config"])
-            total_links = sum(len(links) for links in result.values())
-            print(f"   {test_case['name']}: {total_links} links found")
-
-        except Exception as e:
-            print(f"   ❌ {test_case['name']} failed: {e}")
-
-    # Test 3: Performance options (concurrency, timeout, max_links)
-    print("\n🔍 Testing performance options...")
-
-    perf_configs = [
-        {
-            "name": "Low concurrency",
-            "config": {
-                "concurrency": 1,
-                "timeout": 10,
-                "max_links": 50,
-                "include_internal": True,
-                "include_external": True
-            }
-        },
-        {
-            "name": "High concurrency",
-            "config": {
-                "concurrency": 5,
-                "timeout": 15,
-                "max_links": 200,
-                "include_internal": True,
-                "include_external": True
-            }
-        },
-        {
-            "name": "Very limited",
-            "config": {
-                "concurrency": 1,
-                "timeout": 2,
-                "max_links": 5,
-                "include_internal": True,
-                "include_external": True
-            }
-        }
-    ]
-
-    for test_case in perf_configs:
-        try:
-            start_time = time.time()
-            result = tester.analyze_links(test_url, test_case["config"])
-            end_time = time.time()
-
-            total_links = sum(len(links) for links in result.values())
-            duration = end_time - start_time
-
-            print(f"   {test_case['name']}: {total_links} links in {duration:.2f}s")
-
-            # Verify max_links constraint
-            if total_links > test_case["config"]["max_links"]:
-                print(f"   ⚠️  Found {total_links} links, expected max {test_case['config']['max_links']}")
-
-        except Exception as e:
-            print(f"   ❌ {test_case['name']} failed: {e}")
-
-    # Test 4: Scoring and filtering options
-    print("\n🔍 Testing scoring and filtering options...")
-
-    scoring_configs = [
-        {
-            "name": "No score threshold",
-            "config": {
-                "score_threshold": None,
-                "include_internal": True,
-                "include_external": True
-            }
-        },
-        {
-            "name": "Low score threshold",
-            "config": {
-                "score_threshold": 0.1,
-                "include_internal": True,
-                "include_external": True
-            }
-        },
-        {
-            "name": "High score threshold",
-            "config": {
-                "score_threshold": 0.8,
-                "include_internal": True,
-                "include_external": True
-            }
-        },
-        {
-            "name": "With query for contextual scoring",
-            "config": {
-                "query": "test links",
-                "score_threshold": 0.3,
-                "include_internal": True,
-                "include_external": True
-            }
-        }
-    ]
-
-    for test_case in scoring_configs:
-        try:
-            result = tester.analyze_links(test_url, test_case["config"])
-            total_links = sum(len(links) for links in result.values())
-
-            # Check score threshold
-            if test_case["config"]["score_threshold"] is not None:
-                min_score = test_case["config"]["score_threshold"]
-                low_score_links = 0
-
-                for links in result.values():
-                    for link in links:
-                        score = link.get('total_score', link.get('intrinsic_score', 0))
-                        if score is not None and score < min_score:
-                            low_score_links += 1
-
-                if low_score_links > 0:
-                    print(f"   ⚠️  Found {low_score_links} links below threshold {min_score}")
-                else:
-                    print(f"   ✅ All links meet threshold {min_score}")
-
-            print(f"   {test_case['name']}: {total_links} links")
-
-        except Exception as e:
-            print(f"   ❌ {test_case['name']} failed: {e}")
-
-    # Test 5: Verbose mode
-    print("\n🔍 Testing verbose mode...")
-
-    try:
-        result = tester.analyze_links(test_url, {
-            "verbose": True,
-            "include_internal": True,
-            "include_external": True
-        })
-        total_links = sum(len(links) for links in result.values())
-        print(f"   Verbose mode: {total_links} links")
-
-    except Exception as e:
-        print(f"   ❌ Verbose mode failed: {e}")
-
-    print("✅ All configuration options test passed")
-
-
-def test_link_analysis_edge_cases():
-    """Test edge cases and error scenarios for configuration options"""
-    print("\n=== Testing Edge Cases ===")
-
-    tester = LinkAnalysisTester()
-    test_url = "https://httpbin.org/links/10"
-
-    # Test 1: Invalid configuration values
-    print("\n🔍 Testing invalid configuration values...")
-
-    invalid_configs = [
-        {
-            "name": "Negative concurrency",
-            "config": {"concurrency": -1}
-        },
-        {
-            "name": "Zero timeout",
-            "config": {"timeout": 0}
-        },
-        {
-            "name": "Negative max_links",
-            "config": {"max_links": -5}
-        },
-        {
-            "name": "Invalid score threshold (too high)",
-            "config": {"score_threshold": 1.5}
-        },
-        {
-            "name": "Invalid score threshold (too low)",
-            "config": {"score_threshold": -0.1}
-        },
-        {
-            "name": "Both include flags false",
-            "config": {"include_internal": False, "include_external": False}
-        }
-    ]
-
-    for test_case in invalid_configs:
-        try:
-            result = tester.analyze_links(test_url, test_case["config"])
-            print(f"   ⚠️  {test_case['name']}: Expected to fail but succeeded")
-
-        except Exception as e:
-            print(f"   ✅ {test_case['name']}: Correctly failed - {str(e)}")
-
-    # Test 2: Extreme but valid values
-    print("\n🔍 Testing extreme valid values...")
-
-    extreme_configs = [
-        {
-            "name": "Very high concurrency",
-            "config": {
-                "concurrency": 50,
-                "timeout": 30,
-                "max_links": 1000,
-                "include_internal": True,
-                "include_external": True
-            }
-        },
-        {
-            "name": "Very low score threshold",
-            "config": {
-                "score_threshold": 0.0,
-                "include_internal": True,
-                "include_external": True
-            }
-        },
-        {
-            "name": "Very high score threshold",
-            "config": {
-                "score_threshold": 1.0,
-                "include_internal": True,
-                "include_external": True
-            }
-        }
-    ]
-
-    for test_case in extreme_configs:
-        try:
-            result = tester.analyze_links(test_url, test_case["config"])
-            total_links = sum(len(links) for links in result.values())
-            print(f"   ✅ {test_case['name']}: {total_links} links")
-
-        except Exception as e:
-            print(f"   ❌ {test_case['name']} failed: {e}")
-
-    # Test 3: Complex pattern matching
-    print("\n🔍 Testing complex pattern matching...")
-
-    pattern_configs = [
-        {
-            "name": "Multiple include patterns",
-            "config": {
-                "include_patterns": ["*/links/*", "*/test*", "*/httpbin*"],
-                "include_internal": True,
-                "include_external": True
-            }
-        },
-        {
-            "name": "Multiple exclude patterns",
-            "config": {
-                "exclude_patterns": ["*/admin*", "*/login*", "*/logout*", "*/private*"],
-                "include_internal": True,
-                "include_external": True
-            }
-        },
-        {
-            "name": "Overlapping include/exclude patterns",
-            "config": {
-                "include_patterns": ["*"],
-                "exclude_patterns": ["*/admin*", "*/private*"],
-                "include_internal": True,
-                "include_external": True
-            }
-        }
-    ]
-
-    for test_case in pattern_configs:
-        try:
-            result = tester.analyze_links(test_url, test_case["config"])
-            total_links = sum(len(links) for links in result.values())
-            print(f"   {test_case['name']}: {total_links} links")
-
-        except Exception as e:
-            print(f"   ❌ {test_case['name']} failed: {e}")
-
-    print("✅ Edge cases test passed")
-
-
-def test_link_analysis_batch():
-    """Test batch link analysis"""
-    print("\n=== Testing Batch Analysis ===")
-
-    tester = LinkAnalysisTester()
-
-    test_urls = [
-        "https://httpbin.org/links/10",
-        "https://httpbin.org/links/5",
-        "https://httpbin.org/links/2"
-    ]
-
-    try:
-        results = {}
-        for url in test_urls:
-            print(f"🔍 Analyzing: {url}")
-            result = tester.analyze_links(url)
-            results[url] = result
-
-            # Small delay to be respectful
-            time.sleep(0.5)
-
-        print(f"✅ Successfully analyzed {len(results)} URLs")
-
-        for url, result in results.items():
-            total_links = sum(len(links) for links in result.values())
-            print(f"   {url}: {total_links} links")
-
-        print("✅ Batch analysis test passed")
-
-    except Exception as e:
-        print(f"❌ Batch analysis test failed: {str(e)}")
-        raise
-
-
-def run_all_link_analysis_tests():
-    """Run all link analysis tests"""
-    print("🚀 Starting Link Analysis Test Suite")
-    print("=" * 50)
-
-    tests = [
-        test_link_analysis_basic,
-        test_link_analysis_with_config,
-        test_link_analysis_complex_page,
-        test_link_analysis_scoring,
-        test_link_analysis_error_handling,
-        test_link_analysis_performance,
-        test_link_analysis_categorization,
-        test_link_analysis_batch
-    ]
-
-    passed = 0
-    failed = 0
-
-    for test_func in tests:
-        try:
-            test_func()
-            passed += 1
-            print(f"✅ {test_func.__name__} PASSED")
-        except Exception as e:
-            failed += 1
-            print(f"❌ {test_func.__name__} FAILED: {str(e)}")
-
-        print("-" * 50)
-
-    print(f"\n📊 Test Results: {passed} passed, {failed} failed")
-
-    if failed > 0:
-        print("⚠️  Some tests failed, but this may be due to network or server issues")
-        return False
-
-    print("🎉 All tests passed!")
-    return True
-
-
-if __name__ == "__main__":
-    # Check if server is running
-    import socket
-
-    def check_server(host="localhost", port=11234):
-        try:
-            socket.create_connection((host, port), timeout=5)
-            return True
-        except:
-            return False
-
-    if not check_server():
-        print("❌ Server is not running on localhost:11234")
-        print("Please start the Crawl4AI server first:")
-        print("  cd deploy/docker && python server.py")
-        sys.exit(1)
-
-    success = run_all_link_analysis_tests()
-    sys.exit(0 if success else 1)

tests/test_link_analysis_integration.py

@@ -1,169 +0,0 @@
-import requests
-import json
-import time
-import sys
-
-
-def test_links_analyze_endpoint():
-    """Integration test for the /links/analyze endpoint"""
-
-    base_url = "http://localhost:11234"
-
-    # Health check
-    try:
-        health_response = requests.get(f"{base_url}/health", timeout=5)
-        if health_response.status_code != 200:
-            print("❌ Server health check failed")
-            return False
-        print("✅ Server health check passed")
-    except Exception as e:
-        print(f"❌ Cannot connect to server: {e}")
-        return False
-
-    # Get auth token
-    token = None
-    try:
-        token_response = requests.post(
-            f"{base_url}/token",
-            json={"email": "test@example.com"},
-            timeout=5
-        )
-        if token_response.status_code == 200:
-            token = token_response.json()["access_token"]
-            print("✅ Authentication token obtained")
-    except Exception as e:
-        print(f"⚠️  Could not get auth token: {e}")
-
-    # Test the links/analyze endpoint
-    headers = {"Content-Type": "application/json"}
-    if token:
-        headers["Authorization"] = f"Bearer {token}"
-
-    # Test 1: Basic request
-    print("\n🔍 Testing basic link analysis...")
-    test_data = {
-        "url": "https://httpbin.org/links/10",
-        "config": {
-            "include_internal": True,
-            "include_external": True,
-            "max_links": 50,
-            "verbose": True
-        }
-    }
-
-    try:
-        response = requests.post(
-            f"{base_url}/links/analyze",
-            headers=headers,
-            json=test_data,
-            timeout=30
-        )
-
-        if response.status_code == 200:
-            result = response.json()
-            print("✅ Basic link analysis successful")
-            print(f"📄 Response structure: {list(result.keys())}")
-
-            # Verify response structure
-            total_links = sum(len(links) for links in result.values())
-            print(f"📊 Found {total_links} total links")
-
-            # Debug: Show what was actually returned
-            if total_links == 0:
-                print("⚠️  No links found - showing full response:")
-                print(json.dumps(result, indent=2))
-
-            # Check for expected categories
-            found_categories = []
-            for category in ['internal', 'external', 'social', 'download', 'email', 'phone']:
-                if category in result and result[category]:
-                    found_categories.append(category)
-
-            print(f"📂 Found categories: {found_categories}")
-
-            # Verify link objects have required fields
-            if total_links > 0:
-                sample_found = False
-                for category, links in result.items():
-                    if links:
-                        sample_link = links[0]
-                        if 'href' in sample_link and 'total_score' in sample_link:
-                            sample_found = True
-                            break
-
-                if sample_found:
-                    print("✅ Link objects have required fields")
-                else:
-                    print("⚠️  Link objects missing required fields")
-
-        else:
-            print(f"❌ Basic link analysis failed: {response.status_code}")
-            print(f"Response: {response.text}")
-            return False
-
-    except Exception as e:
-        print(f"❌ Basic link analysis error: {e}")
-        return False
-
-    # Test 2: With configuration
-    print("\n🔍 Testing link analysis with configuration...")
-    test_data_with_config = {
-        "url": "https://httpbin.org/links/10",
-        "config": {
-            "include_internal": True,
-            "include_external": True,
-            "max_links": 50,
-            "timeout": 10,
-            "verbose": True
-        }
-    }
-
-    try:
-        response = requests.post(
-            f"{base_url}/links/analyze",
-            headers=headers,
-            json=test_data_with_config,
-            timeout=30
-        )
-
-        if response.status_code == 200:
-            result = response.json()
-            total_links = sum(len(links) for links in result.values())
-            print(f"✅ Link analysis with config successful ({total_links} links)")
-        else:
-            print(f"❌ Link analysis with config failed: {response.status_code}")
-            return False
-
-    except Exception as e:
-        print(f"❌ Link analysis with config error: {e}")
-        return False
-
-    # Test 3: Error handling
-    print("\n🔍 Testing error handling...")
-    invalid_data = {
-        "url": "not-a-valid-url"
-    }
-
-    try:
-        response = requests.post(
-            f"{base_url}/links/analyze",
-            headers=headers,
-            json=invalid_data,
-            timeout=30
-        )
-
-        if response.status_code >= 400:
-            print("✅ Error handling works correctly")
-        else:
-            print("⚠️  Expected error for invalid URL, but got success")
-
-    except Exception as e:
-        print(f"✅ Error handling caught exception: {e}")
-
-    print("\n🎉 All integration tests passed!")
-    return True
-
-
-if __name__ == "__main__":
-    success = test_links_analyze_endpoint()
-    sys.exit(0 if success else 1)

tests/test_url_discovery.py

@@ -1,193 +0,0 @@
-#!/usr/bin/env python3
-"""
-Test script for the new /urls/discover endpoint in Crawl4AI Docker API.
-"""
-
-import asyncio
-import httpx
-import json
-from rich.console import Console
-from rich.panel import Panel
-from rich.syntax import Syntax
-
-console = Console()
-
-# Configuration
-BASE_URL = "http://localhost:11235"
-TEST_DOMAIN = "docs.crawl4ai.com"
-
-async def check_server_health(client: httpx.AsyncClient) -> bool:
-    """Check if the server is healthy."""
-    console.print("[bold cyan]Checking server health...[/]", end="")
-    try:
-        response = await client.get("/health", timeout=10.0)
-        response.raise_for_status()
-        console.print(" [bold green]✓ Server is healthy![/]")
-        return True
-    except Exception as e:
-        console.print(f"\n[bold red]✗ Server health check failed: {e}[/]")
-        console.print(f"Is the server running at {BASE_URL}?")
-        return False
-
-def print_request(endpoint: str, payload: dict, title: str = "Request"):
-    """Pretty print the request."""
-    syntax = Syntax(json.dumps(payload, indent=2), "json", theme="monokai")
-    console.print(Panel.fit(
-        f"[cyan]POST {endpoint}[/cyan]\n{syntax}",
-        title=f"[bold blue]{title}[/]",
-        border_style="blue"
-    ))
-
-def print_response(response_data: dict, title: str = "Response"):
-    """Pretty print the response."""
-    syntax = Syntax(json.dumps(response_data, indent=2), "json", theme="monokai")
-    console.print(Panel.fit(
-        syntax,
-        title=f"[bold green]{title}[/]",
-        border_style="green"
-    ))
-
-async def test_urls_discover_basic():
-    """Test basic URL discovery functionality."""
-    console.print("\n[bold yellow]Testing URL Discovery Endpoint[/bold yellow]")
-    
-    async with httpx.AsyncClient(base_url=BASE_URL, timeout=30.0) as client:
-        # Check server health first
-        if not await check_server_health(client):
-            return False
-        
-        # Test 1: Basic discovery with sitemap
-        console.print("\n[cyan]Test 1: Basic URL discovery from sitemap[/cyan]")
-        
-        payload = {
-            "domain": TEST_DOMAIN,
-            "seeding_config": {
-                "source": "sitemap",
-                "max_urls": 5
-            }
-        }
-        
-        print_request("/urls/discover", payload, "Basic Discovery Request")
-        
-        try:
-            response = await client.post("/urls/discover", json=payload)
-            response.raise_for_status()
-            response_data = response.json()
-            
-            print_response(response_data, "Basic Discovery Response")
-            
-            # Validate response structure
-            if isinstance(response_data, list):
-                console.print(f"[green]✓ Discovered {len(response_data)} URLs[/green]")
-                return True
-            else:
-                console.print(f"[red]✗ Expected list, got {type(response_data)}[/red]")
-                return False
-                
-        except httpx.HTTPStatusError as e:
-            console.print(f"[red]✗ HTTP Error: {e.response.status_code} - {e.response.text}[/red]")
-            return False
-        except Exception as e:
-            console.print(f"[red]✗ Error: {e}[/red]")
-            return False
-
-async def test_urls_discover_invalid_config():
-    """Test URL discovery with invalid configuration."""
-    console.print("\n[cyan]Test 2: URL discovery with invalid configuration[/cyan]")
-    
-    async with httpx.AsyncClient(base_url=BASE_URL, timeout=30.0) as client:
-        payload = {
-            "domain": TEST_DOMAIN,
-            "seeding_config": {
-                "source": "invalid_source",  # Invalid source
-                "max_urls": 5
-            }
-        }
-        
-        print_request("/urls/discover", payload, "Invalid Config Request")
-        
-        try:
-            response = await client.post("/urls/discover", json=payload)
-            
-            if response.status_code == 500:
-                console.print("[green]✓ Server correctly rejected invalid config with 500 error[/green]")
-                return True
-            else:
-                console.print(f"[yellow]? Expected 500 error, got {response.status_code}[/yellow]")
-                response_data = response.json()
-                print_response(response_data, "Unexpected Response")
-                return False
-                
-        except Exception as e:
-            console.print(f"[red]✗ Unexpected error: {e}[/red]")
-            return False
-
-async def test_urls_discover_with_filtering():
-    """Test URL discovery with advanced filtering."""
-    console.print("\n[cyan]Test 3: URL discovery with filtering and metadata[/cyan]")
-    
-    async with httpx.AsyncClient(base_url=BASE_URL, timeout=60.0) as client:
-        payload = {
-            "domain": TEST_DOMAIN,
-            "seeding_config": {
-                "source": "sitemap",
-                "pattern": "*/docs/*",  # Filter to docs URLs only
-                "extract_head": True,   # Extract metadata
-                "max_urls": 3
-            }
-        }
-        
-        print_request("/urls/discover", payload, "Filtered Discovery Request")
-        
-        try:
-            response = await client.post("/urls/discover", json=payload)
-            response.raise_for_status()
-            response_data = response.json()
-            
-            print_response(response_data, "Filtered Discovery Response")
-            
-            # Validate response structure with metadata
-            if isinstance(response_data, list) and len(response_data) > 0:
-                sample_url = response_data[0]
-                if "url" in sample_url:
-                    console.print(f"[green]✓ Discovered {len(response_data)} filtered URLs with metadata[/green]")
-                    return True
-                else:
-                    console.print(f"[red]✗ URL objects missing expected fields[/red]")
-                    return False
-            else:
-                console.print(f"[yellow]? No URLs found with filter pattern[/yellow]")
-                return True  # This could be expected
-                
-        except httpx.HTTPStatusError as e:
-            console.print(f"[red]✗ HTTP Error: {e.response.status_code} - {e.response.text}[/red]")
-            return False
-        except Exception as e:
-            console.print(f"[red]✗ Error: {e}[/red]")
-            return False
-
-async def main():
-    """Run all tests."""
-    console.print("[bold cyan]🔍 URL Discovery Endpoint Tests[/bold cyan]")
-    
-    results = []
-    
-    # Run tests
-    results.append(await test_urls_discover_basic())
-    results.append(await test_urls_discover_invalid_config())
-    results.append(await test_urls_discover_with_filtering())
-    
-    # Summary
-    console.print("\n[bold cyan]Test Summary[/bold cyan]")
-    passed = sum(results)
-    total = len(results)
-    
-    if passed == total:
-        console.print(f"[bold green]✓ All {total} tests passed![/bold green]")
-    else:
-        console.print(f"[bold yellow]⚠ {passed}/{total} tests passed[/bold yellow]")
-    
-    return passed == total
-
-if __name__ == "__main__":
-    asyncio.run(main())

tests/test_url_discovery_e2e.py

@@ -1,286 +0,0 @@
-#!/usr/bin/env python3
-"""
-End-to-end tests for the URL Discovery endpoint.
-
-This test suite verifies the complete functionality of the /urls/discover endpoint
-including happy path scenarios and error handling.
-"""
-
-import asyncio
-import httpx
-import json
-import pytest
-from typing import Dict, Any
-
-# Test configuration
-BASE_URL = "http://localhost:11235"
-TEST_TIMEOUT = 30.0
-
-
-class TestURLDiscoveryEndpoint:
-    """End-to-end test suite for URL Discovery endpoint."""
-    
-    @pytest.fixture
-    async def client(self):
-        """Create an async HTTP client for testing."""
-        async with httpx.AsyncClient(base_url=BASE_URL, timeout=TEST_TIMEOUT) as client:
-            yield client
-    
-    async def test_server_health(self, client):
-        """Test that the server is healthy before running other tests."""
-        response = await client.get("/health")
-        assert response.status_code == 200
-        data = response.json()
-        assert data["status"] == "ok"
-    
-    async def test_endpoint_exists(self, client):
-        """Test that the /urls/discover endpoint exists and is documented."""
-        # Check OpenAPI spec includes our endpoint
-        response = await client.get("/openapi.json")
-        assert response.status_code == 200
-        
-        openapi_spec = response.json()
-        assert "/urls/discover" in openapi_spec["paths"]
-        
-        endpoint_spec = openapi_spec["paths"]["/urls/discover"]
-        assert "post" in endpoint_spec
-        assert endpoint_spec["post"]["summary"] == "URL Discovery and Seeding"
-    
-    async def test_basic_url_discovery_happy_path(self, client):
-        """Test basic URL discovery with minimal configuration."""
-        request_data = {
-            "domain": "example.com",
-            "seeding_config": {
-                "source": "sitemap",
-                "max_urls": 5
-            }
-        }
-        
-        response = await client.post("/urls/discover", json=request_data)
-        assert response.status_code == 200
-        
-        data = response.json()
-        assert isinstance(data, list)
-        # Note: We don't assert length > 0 because URL discovery 
-        # may legitimately return empty results
-    
-    async def test_minimal_request_with_defaults(self, client):
-        """Test that minimal request works with default seeding_config."""
-        request_data = {
-            "domain": "example.com"
-        }
-        
-        response = await client.post("/urls/discover", json=request_data)
-        assert response.status_code == 200
-        
-        data = response.json()
-        assert isinstance(data, list)
-    
-    async def test_advanced_configuration(self, client):
-        """Test advanced configuration options."""
-        request_data = {
-            "domain": "example.com",
-            "seeding_config": {
-                "source": "sitemap+cc",
-                "pattern": "*/docs/*",
-                "extract_head": True,
-                "max_urls": 3,
-                "live_check": True,
-                "concurrency": 50,
-                "hits_per_sec": 5,
-                "verbose": True
-            }
-        }
-        
-        response = await client.post("/urls/discover", json=request_data)
-        assert response.status_code == 200
-        
-        data = response.json()
-        assert isinstance(data, list)
-        
-        # If URLs are returned, they should have the expected structure
-        for url_obj in data:
-            assert isinstance(url_obj, dict)
-            # Should have at least a URL field
-            assert "url" in url_obj
-    
-    async def test_bm25_scoring_configuration(self, client):
-        """Test BM25 relevance scoring configuration."""
-        request_data = {
-            "domain": "example.com",
-            "seeding_config": {
-                "source": "sitemap",
-                "extract_head": True,  # Required for scoring
-                "query": "documentation",
-                "scoring_method": "bm25",
-                "score_threshold": 0.1,
-                "max_urls": 5
-            }
-        }
-        
-        response = await client.post("/urls/discover", json=request_data)
-        assert response.status_code == 200
-        
-        data = response.json()
-        assert isinstance(data, list)
-        
-        # If URLs are returned with scoring, check structure
-        for url_obj in data:
-            assert isinstance(url_obj, dict)
-            assert "url" in url_obj
-            # Scoring may or may not add score field depending on implementation
-    
-    async def test_missing_required_domain_field(self, client):
-        """Test error handling when required domain field is missing."""
-        request_data = {
-            "seeding_config": {
-                "source": "sitemap",
-                "max_urls": 5
-            }
-        }
-        
-        response = await client.post("/urls/discover", json=request_data)
-        assert response.status_code == 422  # Validation error
-        
-        error_data = response.json()
-        assert "detail" in error_data
-        assert any("domain" in str(error).lower() for error in error_data["detail"])
-    
-    async def test_invalid_request_body_structure(self, client):
-        """Test error handling with completely invalid request body."""
-        invalid_request = {
-            "invalid_field": "test_value",
-            "another_invalid": 123
-        }
-        
-        response = await client.post("/urls/discover", json=invalid_request)
-        assert response.status_code == 422  # Validation error
-        
-        error_data = response.json()
-        assert "detail" in error_data
-    
-    async def test_invalid_seeding_config_parameters(self, client):
-        """Test handling of invalid seeding configuration parameters."""
-        request_data = {
-            "domain": "example.com",
-            "seeding_config": {
-                "source": "invalid_source",  # Invalid source
-                "max_urls": "not_a_number"   # Invalid type
-            }
-        }
-        
-        response = await client.post("/urls/discover", json=request_data)
-        # The endpoint should handle this gracefully
-        # It may return 200 with empty results or 500 with error details
-        assert response.status_code in [200, 500]
-        
-        if response.status_code == 200:
-            data = response.json()
-            assert isinstance(data, list)
-            # May be empty due to invalid config
-        else:
-            # Should have error details
-            error_data = response.json()
-            assert "detail" in error_data
-    
-    async def test_empty_seeding_config(self, client):
-        """Test with empty seeding_config object."""
-        request_data = {
-            "domain": "example.com",
-            "seeding_config": {}
-        }
-        
-        response = await client.post("/urls/discover", json=request_data)
-        assert response.status_code == 200
-        
-        data = response.json()
-        assert isinstance(data, list)
-    
-    async def test_response_structure_consistency(self, client):
-        """Test that response structure is consistent."""
-        request_data = {
-            "domain": "example.com",
-            "seeding_config": {
-                "source": "sitemap",
-                "max_urls": 1
-            }
-        }
-        
-        # Make multiple requests to ensure consistency
-        for _ in range(3):
-            response = await client.post("/urls/discover", json=request_data)
-            assert response.status_code == 200
-            
-            data = response.json()
-            assert isinstance(data, list)
-            
-            # If there are results, check they have consistent structure
-            for url_obj in data:
-                assert isinstance(url_obj, dict)
-                assert "url" in url_obj
-    
-    async def test_content_type_validation(self, client):
-        """Test that endpoint requires JSON content type."""
-        # Test with wrong content type
-        response = await client.post(
-            "/urls/discover",
-            content="domain=example.com",
-            headers={"Content-Type": "application/x-www-form-urlencoded"}
-        )
-        assert response.status_code == 422
-
-
-# Standalone test runner for when pytest is not available
-async def run_tests_standalone():
-    """Run tests without pytest framework."""
-    print("🧪 Running URL Discovery Endpoint Tests")
-    print("=" * 50)
-    
-    # Check server health first
-    async with httpx.AsyncClient(base_url=BASE_URL, timeout=TEST_TIMEOUT) as client:
-        try:
-            response = await client.get("/health")
-            assert response.status_code == 200
-            print("✅ Server health check passed")
-        except Exception as e:
-            print(f"❌ Server health check failed: {e}")
-            return False
-    
-    test_suite = TestURLDiscoveryEndpoint()
-    
-    # Run tests manually
-    tests = [
-        ("Endpoint exists", test_suite.test_endpoint_exists),
-        ("Basic URL discovery", test_suite.test_basic_url_discovery_happy_path),
-        ("Minimal request", test_suite.test_minimal_request_with_defaults),
-        ("Advanced configuration", test_suite.test_advanced_configuration),
-        ("BM25 scoring", test_suite.test_bm25_scoring_configuration),
-        ("Missing domain error", test_suite.test_missing_required_domain_field),
-        ("Invalid request body", test_suite.test_invalid_request_body_structure),
-        ("Invalid config handling", test_suite.test_invalid_seeding_config_parameters),
-        ("Empty config", test_suite.test_empty_seeding_config),
-        ("Response consistency", test_suite.test_response_structure_consistency),
-        ("Content type validation", test_suite.test_content_type_validation),
-    ]
-    
-    passed = 0
-    failed = 0
-    
-    async with httpx.AsyncClient(base_url=BASE_URL, timeout=TEST_TIMEOUT) as client:
-        for test_name, test_func in tests:
-            try:
-                await test_func(client)
-                print(f"✅ {test_name}")
-                passed += 1
-            except Exception as e:
-                print(f"❌ {test_name}: {e}")
-                failed += 1
-    
-    print(f"\n📊 Test Results: {passed} passed, {failed} failed")
-    return failed == 0
-
-
-if __name__ == "__main__":
-    # Run tests standalone
-    success = asyncio.run(run_tests_standalone())
-    exit(0 if success else 1)

tests/test_virtual_scroll_api.py

@@ -1,170 +0,0 @@
-#!/usr/bin/env python3
-"""
-Test script for VirtualScrollConfig with the /crawl API endpoint
-"""
-
-import requests
-import json
-
-def test_virtual_scroll_api():
-    """Test the /crawl endpoint with VirtualScrollConfig"""
-
-    # Create a simple HTML page with virtual scroll for testing
-    test_html = '''
-    <html>
-    <head>
-        <style>
-            #container {
-                height: 300px;
-                overflow-y: auto;
-                border: 1px solid #ccc;
-            }
-            .item {
-                height: 30px;
-                padding: 5px;
-                border-bottom: 1px solid #eee;
-            }
-        </style>
-    </head>
-    <body>
-        <h1>Virtual Scroll Test</h1>
-        <div id="container">
-            <div class="item">Item 1</div>
-            <div class="item">Item 2</div>
-            <div class="item">Item 3</div>
-            <div class="item">Item 4</div>
-            <div class="item">Item 5</div>
-        </div>
-        <script>
-            // Simple script to simulate virtual scroll
-            const container = document.getElementById('container');
-            let itemCount = 5;
-
-            // Add more items when scrolling
-            container.addEventListener('scroll', function() {
-                if (container.scrollTop + container.clientHeight >= container.scrollHeight - 10) {
-                    for (let i = 0; i < 5; i++) {
-                        itemCount++;
-                        const newItem = document.createElement('div');
-                        newItem.className = 'item';
-                        newItem.textContent = `Item ${itemCount}`;
-                        container.appendChild(newItem);
-                    }
-                }
-            });
-
-            // Initial scroll to trigger loading
-            setTimeout(() => {
-                container.scrollTop = container.scrollHeight;
-            }, 100);
-        </script>
-    </body>
-    </html>
-    '''
-
-    # Save the HTML to a temporary file and serve it
-    import tempfile
-    import os
-    import http.server
-    import socketserver
-    import threading
-    import time
-
-    # Create temporary HTML file
-    with tempfile.NamedTemporaryFile(mode='w', suffix='.html', delete=False) as f:
-        f.write(test_html)
-        temp_file = f.name
-
-    # Start local server
-    os.chdir(os.path.dirname(temp_file))
-    port = 8080
-
-    class QuietHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
-        def log_message(self, format, *args):
-            pass  # Suppress log messages
-
-    try:
-        with socketserver.TCPServer(("", port), QuietHTTPRequestHandler) as httpd:
-            server_thread = threading.Thread(target=httpd.serve_forever)
-            server_thread.daemon = True
-            server_thread.start()
-
-            time.sleep(0.5)  # Give server time to start
-
-            # Now test the API
-            url = f"http://crawl4ai.com/examples/assets/virtual_scroll_twitter_like.html"
-
-            payload = {
-                "urls": [url],
-                "browser_config": {
-                    "type": "BrowserConfig",
-                    "params": {
-                        "headless": True,
-                        "viewport_width": 1920,
-                        "viewport_height": 1080
-                    }
-                },
-                "crawler_config": {
-                    "type": "CrawlerRunConfig",
-                    "params": {
-                        "virtual_scroll_config": {
-                            "type": "VirtualScrollConfig",
-                            "params": {
-                                "container_selector": "#container",
-                                "scroll_count": 3,
-                                "scroll_by": "container_height",
-                                "wait_after_scroll": 0.5
-                            }
-                        },
-                        "cache_mode": "bypass",
-                        "extraction_strategy": {
-                            "type": "NoExtractionStrategy",
-                            "params": {}
-                        }
-                    }
-                }
-            }
-
-            print("Testing VirtualScrollConfig with /crawl endpoint...")
-            print(f"Test URL: {url}")
-            print("Payload:")
-            print(json.dumps(payload, indent=2))
-
-            response = requests.post(
-                "http://localhost:11234/crawl",
-                json=payload,
-                headers={"Content-Type": "application/json"}
-            )
-
-            print(f"\nResponse Status: {response.status_code}")
-
-            if response.status_code == 200:
-                result = response.json()
-                print("✅ Success! VirtualScrollConfig is working.")
-                print(f"Content length: {len(result[0]['content']['raw_content'])} characters")
-
-                # Check if virtual scroll captured more content
-                if "Item 10" in result[0]['content']['raw_content']:
-                    print("✅ Virtual scroll successfully captured additional content!")
-                else:
-                    print("⚠️  Virtual scroll may not have worked as expected")
-
-                # Print a snippet of the content
-                content_preview = result[0]['content']['raw_content'][:500] + "..."
-                print(f"\nContent preview:\n{content_preview}")
-
-            else:
-                print(f"❌ Error: {response.status_code}")
-                print(f"Response: {response.text}")
-
-    except Exception as e:
-        print(f"❌ Test failed with error: {e}")
-    finally:
-        # Cleanup
-        try:
-            os.unlink(temp_file)
-        except:
-            pass
-
-if __name__ == "__main__":
-    test_virtual_scroll_api()

tests/test_virtual_scroll_api_simple.py

@@ -1,117 +0,0 @@
-#!/usr/bin/env python3
-"""
-Test VirtualScrollConfig with the /crawl API using existing test assets
-"""
-
-import requests
-import json
-import os
-import http.server
-import socketserver
-import threading
-import time
-from pathlib import Path
-
-def test_virtual_scroll_api():
-    """Test the /crawl endpoint with VirtualScrollConfig using test assets"""
-
-    # Use the existing test assets
-    assets_dir = Path(__file__).parent / "docs" / "examples" / "assets"
-    if not assets_dir.exists():
-        print(f"❌ Assets directory not found: {assets_dir}")
-        return
-
-    # Start local server for assets
-    os.chdir(assets_dir)
-    port = 8081
-
-    class QuietHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
-        def log_message(self, format, *args):
-            pass  # Suppress log messages
-
-    try:
-        with socketserver.TCPServer(("", port), QuietHTTPRequestHandler) as httpd:
-            server_thread = threading.Thread(target=httpd.serve_forever)
-            server_thread.daemon = True
-            server_thread.start()
-
-            time.sleep(0.5)  # Give server time to start
-
-            # Test with Twitter-like virtual scroll
-            url = f"http://docs.crawl4ai.com/examples/assets/virtual_scroll_twitter_like.html"
-
-            payload = {
-                "urls": [url],
-                "browser_config": {
-                    "type": "BrowserConfig",
-                    "params": {
-                        "headless": True,
-                        "viewport_width": 1280,
-                        "viewport_height": 800
-                    }
-                },
-                "crawler_config": {
-                    "type": "CrawlerRunConfig",
-                    "params": {
-                        "virtual_scroll_config": {
-                            "type": "VirtualScrollConfig",
-                            "params": {
-                                "container_selector": "#timeline",
-                                "scroll_count": 10,
-                                "scroll_by": "container_height",
-                                "wait_after_scroll": 0.3
-                            }
-                        },
-                        "cache_mode": "bypass",
-                        "extraction_strategy": {
-                            "type": "NoExtractionStrategy",
-                            "params": {}
-                        }
-                    }
-                }
-            }
-
-            print("Testing VirtualScrollConfig with /crawl endpoint...")
-            print(f"Test URL: {url}")
-            print("Payload:")
-            print(json.dumps(payload, indent=2))
-
-            response = requests.post(
-                "http://localhost:11234/crawl",
-                json=payload,
-                headers={"Content-Type": "application/json"},
-                timeout=60  # Longer timeout for virtual scroll
-            )
-
-            print(f"\nResponse Status: {response.status_code}")
-
-            if response.status_code == 200:
-                result = response.json()
-                print("✅ Success! VirtualScrollConfig is working with the API.")
-                print(f"Content length: {len(result[0]['content']['raw_content'])} characters")
-
-                # Check if we captured multiple posts (indicating virtual scroll worked)
-                content = result[0]['content']['raw_content']
-                post_count = content.count("Post #")
-                print(f"Found {post_count} posts in the content")
-
-                if post_count > 5:  # Should capture more than just the initial posts
-                    print("✅ Virtual scroll successfully captured additional content!")
-                else:
-                    print("⚠️  Virtual scroll may not have captured much additional content")
-
-                # Print a snippet of the content
-                content_preview = content[:1000] + "..." if len(content) > 1000 else content
-                print(f"\nContent preview:\n{content_preview}")
-
-            else:
-                print(f"❌ Error: {response.status_code}")
-                print(f"Response: {response.text}")
-
-    except requests.exceptions.Timeout:
-        print("❌ Request timed out - virtual scroll may be taking too long")
-    except Exception as e:
-        print(f"❌ Test failed with error: {e}")
-
-if __name__ == "__main__":
-    test_virtual_scroll_api()