docs: rename Docker deployment to self-hosting guide with comprehensive monitoring documentation

Major documentation restructuring to emphasize self-hosting capabilities and fully document the real-time monitoring system.

Changes:
- Renamed docker-deployment.md → self-hosting.md to better reflect the value proposition
- Updated mkdocs.yml navigation to "Self-Hosting Guide"
- Completely rewrote introduction emphasizing self-hosting benefits:
  * Data privacy and ownership
  * Cost control and transparency
  * Performance and security advantages
  * Full customization capabilities

- Expanded "Metrics & Monitoring" → "Real-time Monitoring & Operations" with:
  * Monitoring Dashboard section documenting the /monitor UI
  * Complete feature breakdown (system health, requests, browsers, janitor, errors)
  * Monitor API Endpoints with all REST endpoints and examples
  * WebSocket Streaming integration guide with Python examples
  * Control Actions for manual browser management
  * Production Integration patterns (Prometheus, custom dashboards, alerting)
  * Key production metrics to track

- Enhanced summary section:
  * What users learned checklist
  * Why self-hosting matters
  * Clear next steps
  * Key resources with monitoring dashboard URL

The monitoring dashboard built 2-3 weeks ago is now fully documented and discoverable.
Users will understand they have complete operational visibility at http://localhost:11235/monitor
with real-time updates, browser pool management, and programmatic control via REST/WebSocket APIs.

This positions Crawl4AI as an enterprise-grade self-hosting solution with DevOps-level
monitoring capabilities, not just a Docker deployment.

.githooks/pre-commit

@@ -1,31 +0,0 @@
-#!/bin/bash
-# Pre-commit hook: Auto-sync cnode files when cnode source is modified
-
-# Colors
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-NC='\033[0m'
-
-# Check if cnode source files are being committed
-CNODE_FILES_CHANGED=$(git diff --cached --name-only | grep -E "deploy/docker/(cnode_cli|server_manager)\.py")
-
-if [ -n "$CNODE_FILES_CHANGED" ]; then
-    echo -e "${YELLOW}🔄 cnode source files modified, auto-syncing to package...${NC}"
-
-    # Run sync script
-    if [ -f "deploy/installer/sync-cnode.sh" ]; then
-        bash deploy/installer/sync-cnode.sh
-
-        # Stage the synced files
-        git add deploy/installer/cnode_pkg/cli.py
-        git add deploy/installer/cnode_pkg/server_manager.py
-
-        echo -e "${GREEN}✅ cnode package synced and staged${NC}"
-    else
-        echo -e "${RED}❌ Error: sync-cnode.sh not found${NC}"
-        exit 1
-    fi
-fi
-
-exit 0

.github/workflows/docker-release.yml

@@ -1,81 +0,0 @@
-name: Docker Release
-on:
-  release:
-    types: [published]
-  push:
-    tags:
-      - 'docker-rebuild-v*'  # Allow manual Docker rebuilds via tags
-
-jobs:
-  docker:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Extract version from release or tag
-        id: get_version
-        run: |
-          if [ "${{ github.event_name }}" == "release" ]; then
-            # Triggered by release event
-            VERSION="${{ github.event.release.tag_name }}"
-            VERSION=${VERSION#v}  # Remove 'v' prefix
-          else
-            # Triggered by docker-rebuild-v* tag
-            VERSION=${GITHUB_REF#refs/tags/docker-rebuild-v}
-          fi
-          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
-          echo "Building Docker images for version: $VERSION"
-
-      - name: Extract major and minor versions
-        id: versions
-        run: |
-          VERSION=${{ steps.get_version.outputs.VERSION }}
-          MAJOR=$(echo $VERSION | cut -d. -f1)
-          MINOR=$(echo $VERSION | cut -d. -f1-2)
-          echo "MAJOR=$MAJOR" >> $GITHUB_OUTPUT
-          echo "MINOR=$MINOR" >> $GITHUB_OUTPUT
-          echo "Semantic versions - Major: $MAJOR, Minor: $MINOR"
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-
-      - name: Build and push Docker images
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          tags: |
-            unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
-            unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}
-            unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}
-            unclecode/crawl4ai:latest
-          platforms: linux/amd64,linux/arm64
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-      - name: Summary
-        run: |
-          echo "## 🐳 Docker Release Complete!" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### Published Images" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:latest\`" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### Platforms" >> $GITHUB_STEP_SUMMARY
-          echo "- linux/amd64" >> $GITHUB_STEP_SUMMARY
-          echo "- linux/arm64" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### 🚀 Pull Command" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY
-          echo "docker pull unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY

.github/workflows/docs/ARCHITECTURE.md

@@ -1,917 +0,0 @@
-# Workflow Architecture Documentation
-
-## Overview
-
-This document describes the technical architecture of the split release pipeline for Crawl4AI.
-
----
-
-## Architecture Diagram
-
-```
-┌─────────────────────────────────────────────────────────────────┐
-│                         Developer                                │
-│                              │                                   │
-│                              ▼                                   │
-│                    git tag v1.2.3                               │
-│                    git push --tags                              │
-└──────────────────────────────┬──────────────────────────────────┘
-                               │
-                               ▼
-┌─────────────────────────────────────────────────────────────────┐
-│                      GitHub Repository                           │
-│                                                                  │
-│  ┌────────────────────────────────────────────────────────┐   │
-│  │                  Tag Event: v1.2.3                      │   │
-│  └────────────────────────────────────────────────────────┘   │
-│                              │                                   │
-│                              ▼                                   │
-│  ┌────────────────────────────────────────────────────────┐   │
-│  │           release.yml (Release Pipeline)               │   │
-│  │  ┌──────────────────────────────────────────────┐     │   │
-│  │  │ 1. Extract Version                            │     │   │
-│  │  │    v1.2.3 → 1.2.3                            │     │   │
-│  │  └──────────────────────────────────────────────┘     │   │
-│  │  ┌──────────────────────────────────────────────┐     │   │
-│  │  │ 2. Validate Version                           │     │   │
-│  │  │    Tag == __version__.py                      │     │   │
-│  │  └──────────────────────────────────────────────┘     │   │
-│  │  ┌──────────────────────────────────────────────┐     │   │
-│  │  │ 3. Build Python Package                       │     │   │
-│  │  │    - Source dist (.tar.gz)                    │     │   │
-│  │  │    - Wheel (.whl)                             │     │   │
-│  │  └──────────────────────────────────────────────┘     │   │
-│  │  ┌──────────────────────────────────────────────┐     │   │
-│  │  │ 4. Upload to PyPI                             │     │   │
-│  │  │    - Authenticate with token                  │     │   │
-│  │  │    - Upload dist/*                            │     │   │
-│  │  └──────────────────────────────────────────────┘     │   │
-│  │  ┌──────────────────────────────────────────────┐     │   │
-│  │  │ 5. Create GitHub Release                      │     │   │
-│  │  │    - Tag: v1.2.3                              │     │   │
-│  │  │    - Body: Install instructions               │     │   │
-│  │  │    - Status: Published                        │     │   │
-│  │  └──────────────────────────────────────────────┘     │   │
-│  └────────────────────────────────────────────────────────┘   │
-│                              │                                   │
-│                              ▼                                   │
-│  ┌────────────────────────────────────────────────────────┐   │
-│  │         Release Event: published (v1.2.3)              │   │
-│  └────────────────────────────────────────────────────────┘   │
-│                              │                                   │
-│                              ▼                                   │
-│  ┌────────────────────────────────────────────────────────┐   │
-│  │         docker-release.yml (Docker Pipeline)           │   │
-│  │  ┌──────────────────────────────────────────────┐     │   │
-│  │  │ 1. Extract Version from Release               │     │   │
-│  │  │    github.event.release.tag_name → 1.2.3     │     │   │
-│  │  └──────────────────────────────────────────────┘     │   │
-│  │  ┌──────────────────────────────────────────────┐     │   │
-│  │  │ 2. Parse Semantic Versions                    │     │   │
-│  │  │    1.2.3 → Major: 1, Minor: 1.2              │     │   │
-│  │  └──────────────────────────────────────────────┘     │   │
-│  │  ┌──────────────────────────────────────────────┐     │   │
-│  │  │ 3. Setup Multi-Arch Build                     │     │   │
-│  │  │    - Docker Buildx                            │     │   │
-│  │  │    - QEMU emulation                           │     │   │
-│  │  └──────────────────────────────────────────────┘     │   │
-│  │  ┌──────────────────────────────────────────────┐     │   │
-│  │  │ 4. Authenticate Docker Hub                    │     │   │
-│  │  │    - Username: DOCKER_USERNAME                │     │   │
-│  │  │    - Token: DOCKER_TOKEN                      │     │   │
-│  │  └──────────────────────────────────────────────┘     │   │
-│  │  ┌──────────────────────────────────────────────┐     │   │
-│  │  │ 5. Build Multi-Arch Images                    │     │   │
-│  │  │    ┌────────────────┬────────────────┐       │     │   │
-│  │  │    │  linux/amd64   │  linux/arm64   │       │     │   │
-│  │  │    └────────────────┴────────────────┘       │     │   │
-│  │  │    Cache: GitHub Actions (type=gha)          │     │   │
-│  │  └──────────────────────────────────────────────┘     │   │
-│  │  ┌──────────────────────────────────────────────┐     │   │
-│  │  │ 6. Push to Docker Hub                         │     │   │
-│  │  │    Tags:                                      │     │   │
-│  │  │    - unclecode/crawl4ai:1.2.3                │     │   │
-│  │  │    - unclecode/crawl4ai:1.2                  │     │   │
-│  │  │    - unclecode/crawl4ai:1                    │     │   │
-│  │  │    - unclecode/crawl4ai:latest               │     │   │
-│  │  └──────────────────────────────────────────────┘     │   │
-│  └────────────────────────────────────────────────────────┘   │
-└─────────────────────────────────────────────────────────────────┘
-                               │
-                               ▼
-┌─────────────────────────────────────────────────────────────────┐
-│                     External Services                            │
-│                                                                  │
-│  ┌──────────────┐  ┌──────────────┐  ┌──────────────┐         │
-│  │    PyPI      │  │  Docker Hub  │  │   GitHub     │         │
-│  │              │  │              │  │              │         │
-│  │  crawl4ai    │  │ unclecode/   │  │  Releases    │         │
-│  │  1.2.3       │  │ crawl4ai     │  │  v1.2.3      │         │
-│  └──────────────┘  └──────────────┘  └──────────────┘         │
-└─────────────────────────────────────────────────────────────────┘
-```
-
----
-
-## Component Details
-
-### 1. Release Pipeline (release.yml)
-
-#### Purpose
-Fast publication of Python package and GitHub release.
-
-#### Input
-- **Trigger**: Git tag matching `v*` (excluding `test-v*`)
-- **Example**: `v1.2.3`
-
-#### Processing Stages
-
-##### Stage 1: Version Extraction
-```bash
-Input:  refs/tags/v1.2.3
-Output: VERSION=1.2.3
-```
-
-**Implementation**:
-```bash
-TAG_VERSION=${GITHUB_REF#refs/tags/v}  # Remove 'refs/tags/v' prefix
-echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT
-```
-
-##### Stage 2: Version Validation
-```bash
-Input:  TAG_VERSION=1.2.3
-Check:  crawl4ai/__version__.py contains __version__ = "1.2.3"
-Output: Pass/Fail
-```
-
-**Implementation**:
-```bash
-PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)")
-if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then
-  exit 1
-fi
-```
-
-##### Stage 3: Package Build
-```bash
-Input:  Source code + pyproject.toml
-Output: dist/crawl4ai-1.2.3.tar.gz
-        dist/crawl4ai-1.2.3-py3-none-any.whl
-```
-
-**Implementation**:
-```bash
-python -m build
-# Uses build backend defined in pyproject.toml
-```
-
-##### Stage 4: PyPI Upload
-```bash
-Input:  dist/*.{tar.gz,whl}
-Auth:   PYPI_TOKEN
-Output: Package published to PyPI
-```
-
-**Implementation**:
-```bash
-twine upload dist/*
-# Environment:
-#   TWINE_USERNAME: __token__
-#   TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-```
-
-##### Stage 5: GitHub Release Creation
-```bash
-Input:  Tag: v1.2.3
-        Body: Markdown content
-Output: Published GitHub release
-```
-
-**Implementation**:
-```yaml
-uses: softprops/action-gh-release@v2
-with:
-  tag_name: v1.2.3
-  name: Release v1.2.3
-  body: |
-    Installation instructions and changelog
-  draft: false
-  prerelease: false
-```
-
-#### Output
-- **PyPI Package**: https://pypi.org/project/crawl4ai/1.2.3/
-- **GitHub Release**: Published release on repository
-- **Event**: `release.published` (triggers Docker workflow)
-
-#### Timeline
-```
-0:00 - Tag pushed
-0:01 - Checkout + Python setup
-0:02 - Version validation
-0:03 - Package build
-0:04 - PyPI upload starts
-0:06 - PyPI upload complete
-0:07 - GitHub release created
-0:08 - Workflow complete
-```
-
----
-
-### 2. Docker Release Pipeline (docker-release.yml)
-
-#### Purpose
-Build and publish multi-architecture Docker images.
-
-#### Inputs
-
-##### Input 1: Release Event (Automatic)
-```yaml
-Event: release.published
-Data:  github.event.release.tag_name = "v1.2.3"
-```
-
-##### Input 2: Docker Rebuild Tag (Manual)
-```yaml
-Tag: docker-rebuild-v1.2.3
-```
-
-#### Processing Stages
-
-##### Stage 1: Version Detection
-```bash
-# From release event:
-VERSION = github.event.release.tag_name.strip("v")
-# Result: "1.2.3"
-
-# From rebuild tag:
-VERSION = GITHUB_REF.replace("refs/tags/docker-rebuild-v", "")
-# Result: "1.2.3"
-```
-
-##### Stage 2: Semantic Version Parsing
-```bash
-Input:  VERSION=1.2.3
-Output: MAJOR=1
-        MINOR=1.2
-        PATCH=3 (implicit)
-```
-
-**Implementation**:
-```bash
-MAJOR=$(echo $VERSION | cut -d. -f1)    # Extract first component
-MINOR=$(echo $VERSION | cut -d. -f1-2)  # Extract first two components
-```
-
-##### Stage 3: Multi-Architecture Setup
-```yaml
-Setup:
-  - Docker Buildx (multi-platform builder)
-  - QEMU (ARM emulation on x86)
-
-Platforms:
-  - linux/amd64 (x86_64)
-  - linux/arm64 (aarch64)
-```
-
-**Architecture**:
-```
-GitHub Runner (linux/amd64)
-  ├─ Buildx Builder
-  │   ├─ Native: Build linux/amd64 image
-  │   └─ QEMU: Emulate ARM to build linux/arm64 image
-  └─ Generate manifest list (points to both images)
-```
-
-##### Stage 4: Docker Hub Authentication
-```bash
-Input:  DOCKER_USERNAME
-        DOCKER_TOKEN
-Output: Authenticated Docker client
-```
-
-##### Stage 5: Build with Cache
-```yaml
-Cache Configuration:
-  cache-from: type=gha           # Read from GitHub Actions cache
-  cache-to: type=gha,mode=max    # Write all layers
-
-Cache Key Components:
-  - Workflow file path
-  - Branch name
-  - Architecture (amd64/arm64)
-```
-
-**Cache Hierarchy**:
-```
-Cache Entry: main/docker-release.yml/linux-amd64
-  ├─ Layer: sha256:abc123... (FROM python:3.12)
-  ├─ Layer: sha256:def456... (RUN apt-get update)
-  ├─ Layer: sha256:ghi789... (COPY requirements.txt)
-  ├─ Layer: sha256:jkl012... (RUN pip install)
-  └─ Layer: sha256:mno345... (COPY . /app)
-
-Cache Hit/Miss Logic:
-  - If layer input unchanged → cache hit → skip build
-  - If layer input changed → cache miss → rebuild + all subsequent layers
-```
-
-##### Stage 6: Tag Generation
-```bash
-Input:  VERSION=1.2.3, MAJOR=1, MINOR=1.2
-
-Output Tags:
-  - unclecode/crawl4ai:1.2.3    (exact version)
-  - unclecode/crawl4ai:1.2      (minor version)
-  - unclecode/crawl4ai:1        (major version)
-  - unclecode/crawl4ai:latest   (latest stable)
-```
-
-**Tag Strategy**:
-- All tags point to same image SHA
-- Users can pin to desired stability level
-- Pushing new version updates `1`, `1.2`, and `latest` automatically
-
-##### Stage 7: Push to Registry
-```bash
-For each tag:
-  For each platform (amd64, arm64):
-    Push image to Docker Hub
-
-Create manifest list:
-  Manifest: unclecode/crawl4ai:1.2.3
-    ├─ linux/amd64: sha256:abc...
-    └─ linux/arm64: sha256:def...
-
-Docker CLI automatically selects correct platform on pull
-```
-
-#### Output
-- **Docker Images**: 4 tags × 2 platforms = 8 image variants + 4 manifests
-- **Docker Hub**: https://hub.docker.com/r/unclecode/crawl4ai/tags
-
-#### Timeline
-
-**Cold Cache (First Build)**:
-```
-0:00 - Release event received
-0:01 - Checkout + Buildx setup
-0:02 - Docker Hub auth
-0:03 - Start build (amd64)
-0:08 - Complete amd64 build
-0:09 - Start build (arm64)
-0:14 - Complete arm64 build
-0:15 - Generate manifests
-0:16 - Push all tags
-0:17 - Workflow complete
-```
-
-**Warm Cache (Code Change Only)**:
-```
-0:00 - Release event received
-0:01 - Checkout + Buildx setup
-0:02 - Docker Hub auth
-0:03 - Start build (amd64) - cache hit for layers 1-4
-0:04 - Complete amd64 build (only layer 5 rebuilt)
-0:05 - Start build (arm64) - cache hit for layers 1-4
-0:06 - Complete arm64 build (only layer 5 rebuilt)
-0:07 - Generate manifests
-0:08 - Push all tags
-0:09 - Workflow complete
-```
-
----
-
-## Data Flow
-
-### Version Information Flow
-
-```
-Developer
-  │
-  ▼
-crawl4ai/__version__.py
-  __version__ = "1.2.3"
-  │
-  ├─► Git Tag
-  │     v1.2.3
-  │       │
-  │       ▼
-  │     release.yml
-  │       │
-  │       ├─► Validation
-  │       │     ✓ Match
-  │       │
-  │       ├─► PyPI Package
-  │       │     crawl4ai==1.2.3
-  │       │
-  │       └─► GitHub Release
-  │             v1.2.3
-  │               │
-  │               ▼
-  │           docker-release.yml
-  │               │
-  │               └─► Docker Tags
-  │                     1.2.3, 1.2, 1, latest
-  │
-  └─► Package Metadata
-        pyproject.toml
-          version = "1.2.3"
-```
-
-### Secrets Flow
-
-```
-GitHub Secrets (Encrypted at Rest)
-  │
-  ├─► PYPI_TOKEN
-  │     │
-  │     ▼
-  │   release.yml
-  │     │
-  │     ▼
-  │   TWINE_PASSWORD env var (masked in logs)
-  │     │
-  │     ▼
-  │   PyPI API (HTTPS)
-  │
-  ├─► DOCKER_USERNAME
-  │     │
-  │     ▼
-  │   docker-release.yml
-  │     │
-  │     ▼
-  │   docker/login-action (masked in logs)
-  │     │
-  │     ▼
-  │   Docker Hub API (HTTPS)
-  │
-  └─► DOCKER_TOKEN
-        │
-        ▼
-      docker-release.yml
-        │
-        ▼
-      docker/login-action (masked in logs)
-        │
-        ▼
-      Docker Hub API (HTTPS)
-```
-
-### Artifact Flow
-
-```
-Source Code
-  │
-  ├─► release.yml
-  │     │
-  │     ▼
-  │   python -m build
-  │     │
-  │     ├─► crawl4ai-1.2.3.tar.gz
-  │     │     │
-  │     │     ▼
-  │     │   PyPI Storage
-  │     │     │
-  │     │     ▼
-  │     │   pip install crawl4ai
-  │     │
-  │     └─► crawl4ai-1.2.3-py3-none-any.whl
-  │           │
-  │           ▼
-  │         PyPI Storage
-  │           │
-  │           ▼
-  │         pip install crawl4ai
-  │
-  └─► docker-release.yml
-        │
-        ▼
-      docker build
-        │
-        ├─► Image: linux/amd64
-        │     │
-        │     └─► Docker Hub
-        │           unclecode/crawl4ai:1.2.3-amd64
-        │
-        └─► Image: linux/arm64
-              │
-              └─► Docker Hub
-                    unclecode/crawl4ai:1.2.3-arm64
-```
-
----
-
-## State Machines
-
-### Release Pipeline State Machine
-
-```
-┌─────────┐
-│  START  │
-└────┬────┘
-     │
-     ▼
-┌──────────────┐
-│ Extract      │
-│ Version      │
-└──────┬───────┘
-       │
-       ▼
-┌──────────────┐      ┌─────────┐
-│ Validate     │─────►│ FAILED  │
-│ Version      │ No   │ (Exit 1)│
-└──────┬───────┘      └─────────┘
-       │ Yes
-       ▼
-┌──────────────┐
-│ Build        │
-│ Package      │
-└──────┬───────┘
-       │
-       ▼
-┌──────────────┐      ┌─────────┐
-│ Upload       │─────►│ FAILED  │
-│ to PyPI      │ Error│ (Exit 1)│
-└──────┬───────┘      └─────────┘
-       │ Success
-       ▼
-┌──────────────┐
-│ Create       │
-│ GH Release   │
-└──────┬───────┘
-       │
-       ▼
-┌──────────────┐
-│  SUCCESS     │
-│ (Emit Event) │
-└──────────────┘
-```
-
-### Docker Pipeline State Machine
-
-```
-┌─────────┐
-│  START  │
-│ (Event) │
-└────┬────┘
-     │
-     ▼
-┌──────────────┐
-│ Detect       │
-│ Version      │
-│ Source       │
-└──────┬───────┘
-       │
-       ▼
-┌──────────────┐
-│ Parse        │
-│ Semantic     │
-│ Versions     │
-└──────┬───────┘
-       │
-       ▼
-┌──────────────┐      ┌─────────┐
-│ Authenticate │─────►│ FAILED  │
-│ Docker Hub   │ Error│ (Exit 1)│
-└──────┬───────┘      └─────────┘
-       │ Success
-       ▼
-┌──────────────┐
-│ Build        │
-│ amd64        │
-└──────┬───────┘
-       │
-       ▼
-┌──────────────┐      ┌─────────┐
-│ Build        │─────►│ FAILED  │
-│ arm64        │ Error│ (Exit 1)│
-└──────┬───────┘      └─────────┘
-       │ Success
-       ▼
-┌──────────────┐
-│ Push All     │
-│ Tags         │
-└──────┬───────┘
-       │
-       ▼
-┌──────────────┐
-│  SUCCESS     │
-└──────────────┘
-```
-
----
-
-## Security Architecture
-
-### Threat Model
-
-#### Threats Mitigated
-
-1. **Secret Exposure**
-   - Mitigation: GitHub Actions secret masking
-   - Evidence: Secrets never appear in logs
-
-2. **Unauthorized Package Upload**
-   - Mitigation: Scoped PyPI tokens
-   - Evidence: Token limited to `crawl4ai` project
-
-3. **Man-in-the-Middle**
-   - Mitigation: HTTPS for all API calls
-   - Evidence: PyPI, Docker Hub, GitHub all use TLS
-
-4. **Supply Chain Tampering**
-   - Mitigation: Immutable artifacts, content checksums
-   - Evidence: PyPI stores SHA256, Docker uses content-addressable storage
-
-#### Trust Boundaries
-
-```
-┌─────────────────────────────────────────┐
-│         Trusted Zone                     │
-│  ┌────────────────────────────────┐    │
-│  │  GitHub Actions Runner         │    │
-│  │  - Ephemeral VM                │    │
-│  │  - Isolated environment        │    │
-│  │  - Access to secrets           │    │
-│  └────────────────────────────────┘    │
-│                │                         │
-│                │ HTTPS (TLS 1.2+)       │
-│                ▼                         │
-└─────────────────────────────────────────┘
-                 │
-    ┌────────────┼────────────┐
-    │            │            │
-    ▼            ▼            ▼
-┌────────┐  ┌─────────┐  ┌──────────┐
-│  PyPI  │  │  Docker │  │  GitHub  │
-│  API   │  │  Hub    │  │  API     │
-└────────┘  └─────────┘  └──────────┘
- External     External     External
-  Service      Service      Service
-```
-
-### Secret Management
-
-#### Secret Lifecycle
-
-```
-Creation (Developer)
-  │
-  ├─► PyPI: Create API token (scoped to project)
-  ├─► Docker Hub: Create access token (read/write)
-  │
-  ▼
-Storage (GitHub)
-  │
-  ├─► Encrypted at rest (AES-256)
-  ├─► Access controlled (repo-scoped)
-  │
-  ▼
-Usage (Workflow)
-  │
-  ├─► Injected as env vars
-  ├─► Masked in logs (GitHub redacts on output)
-  ├─► Never persisted to disk (in-memory only)
-  │
-  ▼
-Transmission (API Call)
-  │
-  ├─► HTTPS only
-  ├─► TLS 1.2+ with strong ciphers
-  │
-  ▼
-Rotation (Manual)
-  │
-  └─► Regenerate on PyPI/Docker Hub
-      Update GitHub secret
-```
-
----
-
-## Performance Characteristics
-
-### Release Pipeline Performance
-
-| Metric | Value | Notes |
-|--------|-------|-------|
-| Cold start | ~2-3 min | First run on new runner |
-| Warm start | ~2-3 min | Minimal caching benefit |
-| PyPI upload | ~30-60 sec | Network-bound |
-| Package build | ~30 sec | CPU-bound |
-| Parallelization | None | Sequential by design |
-
-### Docker Pipeline Performance
-
-| Metric | Cold Cache | Warm Cache (code) | Warm Cache (deps) |
-|--------|-----------|-------------------|-------------------|
-| Total time | 10-15 min | 1-2 min | 3-5 min |
-| amd64 build | 5-7 min | 30-60 sec | 1-2 min |
-| arm64 build | 5-7 min | 30-60 sec | 1-2 min |
-| Push time | 1-2 min | 30 sec | 30 sec |
-| Cache hit rate | 0% | 85% | 60% |
-
-### Cache Performance Model
-
-```python
-def estimate_build_time(changes):
-    base_time = 60  # seconds (setup + push)
-
-    if "Dockerfile" in changes:
-        return base_time + (10 * 60)  # Full rebuild: ~11 min
-    elif "requirements.txt" in changes:
-        return base_time + (3 * 60)   # Deps rebuild: ~4 min
-    elif any(f.endswith(".py") for f in changes):
-        return base_time + 60          # Code only: ~2 min
-    else:
-        return base_time               # No changes: ~1 min
-```
-
----
-
-## Scalability Considerations
-
-### Current Limits
-
-| Resource | Limit | Impact |
-|----------|-------|--------|
-| Workflow concurrency | 20 (default) | Max 20 releases in parallel |
-| Artifact storage | 500 MB/artifact | PyPI packages small (<10 MB) |
-| Cache storage | 10 GB/repo | Docker layers fit comfortably |
-| Workflow run time | 6 hours | Plenty of headroom |
-
-### Scaling Strategies
-
-#### Horizontal Scaling (Multiple Repos)
-```
-crawl4ai (main)
-  ├─ release.yml
-  └─ docker-release.yml
-
-crawl4ai-plugins (separate)
-  ├─ release.yml
-  └─ docker-release.yml
-
-Each repo has independent:
-  - Secrets
-  - Cache (10 GB each)
-  - Concurrency limits (20 each)
-```
-
-#### Vertical Scaling (Larger Runners)
-```yaml
-jobs:
-  docker:
-    runs-on: ubuntu-latest-8-cores  # GitHub-hosted larger runner
-    # 4x faster builds for CPU-bound layers
-```
-
----
-
-## Disaster Recovery
-
-### Failure Scenarios
-
-#### Scenario 1: Release Pipeline Fails
-
-**Failure Point**: PyPI upload fails (network error)
-
-**State**:
-- ✓ Version validated
-- ✓ Package built
-- ✗ PyPI upload
-- ✗ GitHub release
-
-**Recovery**:
-```bash
-# Manual upload
-twine upload dist/*
-
-# Retry workflow (re-run from GitHub Actions UI)
-```
-
-**Prevention**: Add retry logic to PyPI upload
-
-#### Scenario 2: Docker Pipeline Fails
-
-**Failure Point**: ARM build fails (dependency issue)
-
-**State**:
-- ✓ PyPI published
-- ✓ GitHub release created
-- ✓ amd64 image built
-- ✗ arm64 image build
-
-**Recovery**:
-```bash
-# Fix Dockerfile
-git commit -am "fix: ARM build dependency"
-
-# Trigger rebuild
-git tag docker-rebuild-v1.2.3
-git push origin docker-rebuild-v1.2.3
-```
-
-**Impact**: PyPI package available, only Docker ARM users affected
-
-#### Scenario 3: Partial Release
-
-**Failure Point**: GitHub release creation fails
-
-**State**:
-- ✓ PyPI published
-- ✗ GitHub release
-- ✗ Docker images
-
-**Recovery**:
-```bash
-# Create release manually
-gh release create v1.2.3 \
-  --title "Release v1.2.3" \
-  --notes "..."
-
-# This triggers docker-release.yml automatically
-```
-
----
-
-## Monitoring and Observability
-
-### Metrics to Track
-
-#### Release Pipeline
-- Success rate (target: >99%)
-- Duration (target: <3 min)
-- PyPI upload time (target: <60 sec)
-
-#### Docker Pipeline
-- Success rate (target: >95%)
-- Duration (target: <15 min cold, <2 min warm)
-- Cache hit rate (target: >80% for code changes)
-
-### Alerting
-
-**Critical Alerts**:
-- Release pipeline failure (blocks release)
-- PyPI authentication failure (expired token)
-
-**Warning Alerts**:
-- Docker build >15 min (performance degradation)
-- Cache hit rate <50% (cache issue)
-
-### Logging
-
-**GitHub Actions Logs**:
-- Retention: 90 days
-- Downloadable: Yes
-- Searchable: Limited
-
-**Recommended External Logging**:
-```yaml
-- name: Send logs to external service
-  if: failure()
-  run: |
-    curl -X POST https://logs.example.com/api/v1/logs \
-      -H "Content-Type: application/json" \
-      -d "{\"workflow\": \"${{ github.workflow }}\", \"status\": \"failed\"}"
-```
-
----
-
-## Future Enhancements
-
-### Planned Improvements
-
-1. **Automated Changelog Generation**
-   - Use conventional commits
-   - Generate CHANGELOG.md automatically
-
-2. **Pre-release Testing**
-   - Test builds on `test-v*` tags
-   - Upload to TestPyPI
-
-3. **Notification System**
-   - Slack/Discord notifications on release
-   - Email on failure
-
-4. **Performance Optimization**
-   - Parallel Docker builds (amd64 + arm64 simultaneously)
-   - Persistent runners for better caching
-
-5. **Enhanced Validation**
-   - Smoke tests after PyPI upload
-   - Container security scanning
-
----
-
-## References
-
-- [GitHub Actions Architecture](https://docs.github.com/en/actions/learn-github-actions/understanding-github-actions)
-- [Docker Build Cache](https://docs.docker.com/build/cache/)
-- [PyPI API Documentation](https://warehouse.pypa.io/api-reference/)
-
----
-
-**Last Updated**: 2025-01-21
-**Version**: 2.0

.github/workflows/docs/WORKFLOW_REFERENCE.md

@@ -1,287 +0,0 @@
-# Workflow Quick Reference
-
-## Quick Commands
-
-### Standard Release
-```bash
-# 1. Update version
-vim crawl4ai/__version__.py  # Set to "1.2.3"
-
-# 2. Commit and tag
-git add crawl4ai/__version__.py
-git commit -m "chore: bump version to 1.2.3"
-git tag v1.2.3
-git push origin main
-git push origin v1.2.3
-
-# 3. Monitor
-# - PyPI: ~2-3 minutes
-# - Docker: ~1-15 minutes
-```
-
-### Docker Rebuild Only
-```bash
-git tag docker-rebuild-v1.2.3
-git push origin docker-rebuild-v1.2.3
-```
-
-### Delete Tag (Undo Release)
-```bash
-# Local
-git tag -d v1.2.3
-
-# Remote
-git push --delete origin v1.2.3
-
-# GitHub Release
-gh release delete v1.2.3
-```
-
----
-
-## Workflow Triggers
-
-### release.yml
-| Event | Pattern | Example |
-|-------|---------|---------|
-| Tag push | `v*` | `v1.2.3` |
-| Excludes | `test-v*` | `test-v1.2.3` |
-
-### docker-release.yml
-| Event | Pattern | Example |
-|-------|---------|---------|
-| Release published | `release.published` | Automatic |
-| Tag push | `docker-rebuild-v*` | `docker-rebuild-v1.2.3` |
-
----
-
-## Environment Variables
-
-### release.yml
-| Variable | Source | Example |
-|----------|--------|---------|
-| `VERSION` | Git tag | `1.2.3` |
-| `TWINE_USERNAME` | Static | `__token__` |
-| `TWINE_PASSWORD` | Secret | `pypi-Ag...` |
-| `GITHUB_TOKEN` | Auto | `ghp_...` |
-
-### docker-release.yml
-| Variable | Source | Example |
-|----------|--------|---------|
-| `VERSION` | Release/Tag | `1.2.3` |
-| `MAJOR` | Computed | `1` |
-| `MINOR` | Computed | `1.2` |
-| `DOCKER_USERNAME` | Secret | `unclecode` |
-| `DOCKER_TOKEN` | Secret | `dckr_pat_...` |
-
----
-
-## Docker Tags Generated
-
-| Version | Tags Created |
-|---------|-------------|
-| v1.0.0 | `1.0.0`, `1.0`, `1`, `latest` |
-| v1.1.0 | `1.1.0`, `1.1`, `1`, `latest` |
-| v1.2.3 | `1.2.3`, `1.2`, `1`, `latest` |
-| v2.0.0 | `2.0.0`, `2.0`, `2`, `latest` |
-
----
-
-## Workflow Outputs
-
-### release.yml
-| Output | Location | Time |
-|--------|----------|------|
-| PyPI Package | https://pypi.org/project/crawl4ai/ | ~2-3 min |
-| GitHub Release | Repository → Releases | ~2-3 min |
-| Workflow Summary | Actions → Run → Summary | Immediate |
-
-### docker-release.yml
-| Output | Location | Time |
-|--------|----------|------|
-| Docker Images | https://hub.docker.com/r/unclecode/crawl4ai | ~1-15 min |
-| Workflow Summary | Actions → Run → Summary | Immediate |
-
----
-
-## Common Issues
-
-| Issue | Solution |
-|-------|----------|
-| Version mismatch | Update `crawl4ai/__version__.py` to match tag |
-| PyPI 403 Forbidden | Check `PYPI_TOKEN` secret |
-| PyPI 400 File exists | Version already published, increment version |
-| Docker auth failed | Regenerate `DOCKER_TOKEN` |
-| Docker build timeout | Check Dockerfile, review build logs |
-| Cache not working | First build on branch always cold |
-
----
-
-## Secrets Checklist
-
-- [ ] `PYPI_TOKEN` - PyPI API token (project or account scope)
-- [ ] `DOCKER_USERNAME` - Docker Hub username
-- [ ] `DOCKER_TOKEN` - Docker Hub access token (read/write)
-- [ ] `GITHUB_TOKEN` - Auto-provided (no action needed)
-
----
-
-## Workflow Dependencies
-
-### release.yml Dependencies
-```yaml
-Python: 3.12
-Actions:
-  - actions/checkout@v4
-  - actions/setup-python@v5
-  - softprops/action-gh-release@v2
-PyPI Packages:
-  - build
-  - twine
-```
-
-### docker-release.yml Dependencies
-```yaml
-Actions:
-  - actions/checkout@v4
-  - docker/setup-buildx-action@v3
-  - docker/login-action@v3
-  - docker/build-push-action@v5
-Docker:
-  - Buildx
-  - QEMU (for multi-arch)
-```
-
----
-
-## Cache Information
-
-### Type
-- GitHub Actions Cache (`type=gha`)
-
-### Storage
-- **Limit**: 10GB per repository
-- **Retention**: 7 days for unused entries
-- **Cleanup**: Automatic LRU eviction
-
-### Performance
-| Scenario | Cache Hit | Build Time |
-|----------|-----------|------------|
-| First build | 0% | 10-15 min |
-| Code change only | 85% | 1-2 min |
-| Dependency update | 60% | 3-5 min |
-| No changes | 100% | 30-60 sec |
-
----
-
-## Build Platforms
-
-| Platform | Architecture | Devices |
-|----------|--------------|---------|
-| linux/amd64 | x86_64 | Intel/AMD servers, AWS EC2, GCP |
-| linux/arm64 | aarch64 | Apple Silicon, AWS Graviton, Raspberry Pi |
-
----
-
-## Version Validation
-
-### Pre-Tag Checklist
-```bash
-# Check current version
-python -c "from crawl4ai.__version__ import __version__; print(__version__)"
-
-# Verify it matches intended tag
-# If tag is v1.2.3, version should be "1.2.3"
-```
-
-### Post-Release Verification
-```bash
-# PyPI
-pip install crawl4ai==1.2.3
-python -c "import crawl4ai; print(crawl4ai.__version__)"
-
-# Docker
-docker pull unclecode/crawl4ai:1.2.3
-docker run unclecode/crawl4ai:1.2.3 python -c "import crawl4ai; print(crawl4ai.__version__)"
-```
-
----
-
-## Monitoring URLs
-
-| Service | URL |
-|---------|-----|
-| GitHub Actions | `https://github.com/{owner}/{repo}/actions` |
-| PyPI Project | `https://pypi.org/project/crawl4ai/` |
-| Docker Hub | `https://hub.docker.com/r/unclecode/crawl4ai` |
-| GitHub Releases | `https://github.com/{owner}/{repo}/releases` |
-
----
-
-## Rollback Strategy
-
-### PyPI (Cannot Delete)
-```bash
-# Increment patch version
-git tag v1.2.4
-git push origin v1.2.4
-```
-
-### Docker (Can Overwrite)
-```bash
-# Rebuild with fix
-git tag docker-rebuild-v1.2.3
-git push origin docker-rebuild-v1.2.3
-```
-
-### GitHub Release
-```bash
-# Delete release
-gh release delete v1.2.3
-
-# Delete tag
-git push --delete origin v1.2.3
-```
-
----
-
-## Status Badge Markdown
-
-```markdown
-[![Release Pipeline](https://github.com/{owner}/{repo}/actions/workflows/release.yml/badge.svg)](https://github.com/{owner}/{repo}/actions/workflows/release.yml)
-
-[![Docker Release](https://github.com/{owner}/{repo}/actions/workflows/docker-release.yml/badge.svg)](https://github.com/{owner}/{repo}/actions/workflows/docker-release.yml)
-```
-
----
-
-## Timeline Example
-
-```
-0:00 - Push tag v1.2.3
-0:01 - release.yml starts
-0:02 - Version validation passes
-0:03 - Package built
-0:04 - PyPI upload starts
-0:06 - PyPI upload complete ✓
-0:07 - GitHub release created ✓
-0:08 - release.yml complete
-0:08 - docker-release.yml triggered
-0:10 - Docker build starts
-0:12 - amd64 image built (cache hit)
-0:14 - arm64 image built (cache hit)
-0:15 - Images pushed to Docker Hub ✓
-0:16 - docker-release.yml complete
-
-Total: ~16 minutes
-Critical path (PyPI + GitHub): ~8 minutes
-```
-
----
-
-## Contact
-
-For workflow issues:
-1. Check Actions tab for logs
-2. Review this reference
-3. See [README.md](./README.md) for detailed docs

.github/workflows/release.yml

@@ -10,53 +10,53 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write  # Required for creating releases
-
+    
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-
+      
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: '3.12'
-
+      
       - name: Extract version from tag
         id: get_version
         run: |
           TAG_VERSION=${GITHUB_REF#refs/tags/v}
           echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT
           echo "Releasing version: $TAG_VERSION"
-
+      
       - name: Install package dependencies
         run: |
           pip install -e .
-
+      
       - name: Check version consistency
         run: |
           TAG_VERSION=${{ steps.get_version.outputs.VERSION }}
           PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)")
-
+          
           echo "Tag version: $TAG_VERSION"
           echo "Package version: $PACKAGE_VERSION"
-
+          
           if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then
             echo "❌ Version mismatch! Tag: $TAG_VERSION, Package: $PACKAGE_VERSION"
             echo "Please update crawl4ai/__version__.py to match the tag version"
             exit 1
           fi
           echo "✅ Version check passed: $TAG_VERSION"
-
+      
       - name: Install build dependencies
         run: |
           python -m pip install --upgrade pip
           pip install build twine
-
+      
       - name: Build package
         run: python -m build
-
+      
       - name: Check package
         run: twine check dist/*
-
+      
       - name: Upload to PyPI
         env:
           TWINE_USERNAME: __token__
@@ -65,7 +65,37 @@ jobs:
           echo "📦 Uploading to PyPI..."
           twine upload dist/*
           echo "✅ Package uploaded to https://pypi.org/project/crawl4ai/"
-
+      
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      
+      - name: Extract major and minor versions
+        id: versions
+        run: |
+          VERSION=${{ steps.get_version.outputs.VERSION }}
+          MAJOR=$(echo $VERSION | cut -d. -f1)
+          MINOR=$(echo $VERSION | cut -d. -f1-2)
+          echo "MAJOR=$MAJOR" >> $GITHUB_OUTPUT
+          echo "MINOR=$MINOR" >> $GITHUB_OUTPUT
+      
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
+            unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}
+            unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}
+            unclecode/crawl4ai:latest
+          platforms: linux/amd64,linux/arm64
+      
       - name: Create GitHub Release
         uses: softprops/action-gh-release@v2
         with:
@@ -73,29 +103,26 @@ jobs:
           name: Release v${{ steps.get_version.outputs.VERSION }}
           body: |
             ## 🎉 Crawl4AI v${{ steps.get_version.outputs.VERSION }} Released!
-
+            
             ### 📦 Installation
-
+            
             **PyPI:**
             ```bash
             pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}
             ```
-
+            
             **Docker:**
             ```bash
             docker pull unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
             docker pull unclecode/crawl4ai:latest
             ```
-
+            
-            **Note:** Docker images are being built and will be available shortly.
-            Check the [Docker Release workflow](https://github.com/${{ github.repository }}/actions/workflows/docker-release.yml) for build status.
-
             ### 📝 What's Changed
             See [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/main/CHANGELOG.md) for details.
           draft: false
           prerelease: false
           token: ${{ secrets.GITHUB_TOKEN }}
-
+      
       - name: Summary
         run: |
           echo "## 🚀 Release Complete!" >> $GITHUB_STEP_SUMMARY
@@ -105,9 +132,11 @@ jobs:
           echo "- URL: https://pypi.org/project/crawl4ai/" >> $GITHUB_STEP_SUMMARY
           echo "- Install: \`pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### 📋 GitHub Release" >> $GITHUB_STEP_SUMMARY
-          echo "- https://github.com/${{ github.repository }}/releases/tag/v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
           echo "### 🐳 Docker Images" >> $GITHUB_STEP_SUMMARY
-          echo "Docker images are being built in a separate workflow." >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "Check: https://github.com/${{ github.repository }}/actions/workflows/docker-release.yml" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:latest\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 📋 GitHub Release" >> $GITHUB_STEP_SUMMARY
+          echo "https://github.com/${{ github.repository }}/releases/tag/v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY

.github/workflows/release.yml.backup

@@ -1,142 +0,0 @@
-name: Release Pipeline
-on:
-  push:
-    tags:
-      - 'v*'
-      - '!test-v*'  # Exclude test tags
-
-jobs:
-  release:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write  # Required for creating releases
-    
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
-      
-      - name: Extract version from tag
-        id: get_version
-        run: |
-          TAG_VERSION=${GITHUB_REF#refs/tags/v}
-          echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT
-          echo "Releasing version: $TAG_VERSION"
-      
-      - name: Install package dependencies
-        run: |
-          pip install -e .
-      
-      - name: Check version consistency
-        run: |
-          TAG_VERSION=${{ steps.get_version.outputs.VERSION }}
-          PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)")
-          
-          echo "Tag version: $TAG_VERSION"
-          echo "Package version: $PACKAGE_VERSION"
-          
-          if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then
-            echo "❌ Version mismatch! Tag: $TAG_VERSION, Package: $PACKAGE_VERSION"
-            echo "Please update crawl4ai/__version__.py to match the tag version"
-            exit 1
-          fi
-          echo "✅ Version check passed: $TAG_VERSION"
-      
-      - name: Install build dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install build twine
-      
-      - name: Build package
-        run: python -m build
-      
-      - name: Check package
-        run: twine check dist/*
-      
-      - name: Upload to PyPI
-        env:
-          TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-        run: |
-          echo "📦 Uploading to PyPI..."
-          twine upload dist/*
-          echo "✅ Package uploaded to https://pypi.org/project/crawl4ai/"
-      
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-      
-      - name: Extract major and minor versions
-        id: versions
-        run: |
-          VERSION=${{ steps.get_version.outputs.VERSION }}
-          MAJOR=$(echo $VERSION | cut -d. -f1)
-          MINOR=$(echo $VERSION | cut -d. -f1-2)
-          echo "MAJOR=$MAJOR" >> $GITHUB_OUTPUT
-          echo "MINOR=$MINOR" >> $GITHUB_OUTPUT
-      
-      - name: Build and push Docker images
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          tags: |
-            unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
-            unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}
-            unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}
-            unclecode/crawl4ai:latest
-          platforms: linux/amd64,linux/arm64
-      
-      - name: Create GitHub Release
-        uses: softprops/action-gh-release@v2
-        with:
-          tag_name: v${{ steps.get_version.outputs.VERSION }}
-          name: Release v${{ steps.get_version.outputs.VERSION }}
-          body: |
-            ## 🎉 Crawl4AI v${{ steps.get_version.outputs.VERSION }} Released!
-            
-            ### 📦 Installation
-            
-            **PyPI:**
-            ```bash
-            pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}
-            ```
-            
-            **Docker:**
-            ```bash
-            docker pull unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
-            docker pull unclecode/crawl4ai:latest
-            ```
-            
-            ### 📝 What's Changed
-            See [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/main/CHANGELOG.md) for details.
-          draft: false
-          prerelease: false
-          token: ${{ secrets.GITHUB_TOKEN }}
-      
-      - name: Summary
-        run: |
-          echo "## 🚀 Release Complete!" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### 📦 PyPI Package" >> $GITHUB_STEP_SUMMARY
-          echo "- Version: ${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
-          echo "- URL: https://pypi.org/project/crawl4ai/" >> $GITHUB_STEP_SUMMARY
-          echo "- Install: \`pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### 🐳 Docker Images" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:latest\`" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### 📋 GitHub Release" >> $GITHUB_STEP_SUMMARY
-          echo "https://github.com/${{ github.repository }}/releases/tag/v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -185,8 +185,7 @@ Crawl4AI.egg-info/
 requirements0.txt
 a.txt
 
-# Ignore shell scripts globally, but allow test scripts
+*.sh
-# *.sh
 .idea
 docs/examples/.chainlit/
 docs/examples/.chainlit/*
@@ -267,11 +266,11 @@ continue_config.json
 .llm.env
 .private/
 
-.claude/
-
 CLAUDE_MONITOR.md
 CLAUDE.md
 
+.claude/
+
 tests/**/test_site
 tests/**/reports
 tests/**/benchmark_reports
@@ -294,4 +293,3 @@ scripts/
 *.db
 *.rdb
 *.ldb
-.context/

Dockerfile

@@ -1,7 +1,7 @@
 FROM python:3.12-slim-bookworm AS build
 
 # C4ai version
-ARG C4AI_VER=0.7.6
+ARG C4AI_VER=0.7.0-r1
 ENV C4AI_VERSION=$C4AI_VER
 LABEL c4ai.version=$C4AI_VER
 

README.md

@@ -27,13 +27,11 @@
 
 Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community.
 
-[✨ Check out latest update v0.7.6](#-recent-updates)
+[✨ Check out latest update v0.7.4](#-recent-updates)
 
-✨ **New in v0.7.6**: Complete Webhook Infrastructure for Docker Job Queue API! Real-time notifications for both `/crawl/job` and `/llm/job` endpoints with exponential backoff retry, custom headers, and flexible delivery modes. No more polling! [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.6.md)
+✨ New in v0.7.4: Revolutionary LLM Table Extraction with intelligent chunking, enhanced concurrency fixes, memory management refactor, and critical stability improvements. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.4.md)
 
-✨ Recent v0.7.5: Docker Hooks System with function-based API for pipeline customization, Enhanced LLM Integration with custom providers, HTTPS Preservation, and multiple community-reported bug fixes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.5.md)
+✨ Recent v0.7.3: Undetected Browser Support, Multi-URL Configurations, Memory Monitoring, Enhanced Table Extraction, GitHub Sponsors. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.3.md)
-
-✨ Previous v0.7.4: Revolutionary LLM Table Extraction with intelligent chunking, enhanced concurrency fixes, memory management refactor, and critical stability improvements. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.4.md)
 
 <details>
   <summary>🤓 <strong>My Personal Story</strong></summary>
@@ -179,7 +177,7 @@ No rate-limited APIs. No lock-in. Build and own your data pipeline with direct g
 - 📸 **Screenshots**: Capture page screenshots during crawling for debugging or analysis.
 - 📂 **Raw Data Crawling**: Directly process raw HTML (`raw:`) or local files (`file://`).
 - 🔗 **Comprehensive Link Extraction**: Extracts internal, external links, and embedded iframe content.
-- 🛠️ **Customizable Hooks**: Define hooks at every step to customize crawling behavior (supports both string and function-based APIs).
+- 🛠️ **Customizable Hooks**: Define hooks at every step to customize crawling behavior.
 - 💾 **Caching**: Cache data for improved speed and to avoid redundant fetches.
 - 📄 **Metadata Extraction**: Retrieve structured metadata from web pages.
 - 📡 **IFrame Content Extraction**: Seamless extraction from embedded iframe content.
@@ -546,54 +544,6 @@ async def test_news_crawl():
 
 ## ✨ Recent Updates
 
-<details>
-<summary><strong>Version 0.7.5 Release Highlights - The Docker Hooks & Security Update</strong></summary>
-
-- **🔧 Docker Hooks System**: Complete pipeline customization with user-provided Python functions at 8 key points
-- **✨ Function-Based Hooks API (NEW)**: Write hooks as regular Python functions with full IDE support:
-  ```python
-  from crawl4ai import hooks_to_string
-  from crawl4ai.docker_client import Crawl4aiDockerClient
-
-  # Define hooks as regular Python functions
-  async def on_page_context_created(page, context, **kwargs):
-      """Block images to speed up crawling"""
-      await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
-      await page.set_viewport_size({"width": 1920, "height": 1080})
-      return page
-
-  async def before_goto(page, context, url, **kwargs):
-      """Add custom headers"""
-      await page.set_extra_http_headers({'X-Crawl4AI': 'v0.7.5'})
-      return page
-
-  # Option 1: Use hooks_to_string() utility for REST API
-  hooks_code = hooks_to_string({
-      "on_page_context_created": on_page_context_created,
-      "before_goto": before_goto
-  })
-
-  # Option 2: Docker client with automatic conversion (Recommended)
-  client = Crawl4aiDockerClient(base_url="http://localhost:11235")
-  results = await client.crawl(
-      urls=["https://httpbin.org/html"],
-      hooks={
-          "on_page_context_created": on_page_context_created,
-          "before_goto": before_goto
-      }
-  )
-  # ✓ Full IDE support, type checking, and reusability!
-  ```
-
-- **🤖 Enhanced LLM Integration**: Custom providers with temperature control and base_url configuration
-- **🔒 HTTPS Preservation**: Secure internal link handling with `preserve_https_for_internal_links=True`
-- **🐍 Python 3.10+ Support**: Modern language features and enhanced performance
-- **🛠️ Bug Fixes**: Resolved multiple community-reported issues including URL processing, JWT authentication, and proxy configuration
-
-[Full v0.7.5 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.5.md)
-
-</details>
-
 <details>
 <summary><strong>Version 0.7.4 Release Highlights - The Intelligent Table Extraction & Performance Update</strong></summary>
 
@@ -969,36 +919,6 @@ We envision a future where AI is powered by real human knowledge, ensuring data
 For more details, see our [full mission statement](./MISSION.md).
 </details>
 
-## 🌟 Current Sponsors
-
-### 🏢 Enterprise Sponsors & Partners
-
-Our enterprise sponsors and technology partners help scale Crawl4AI to power production-grade data pipelines.
-
-| Company | About | Sponsorship Tier |
-|------|------|----------------------------|
-| <a href="https://dashboard.capsolver.com/passport/register?inviteCode=ESVSECTX5Q23" target="_blank"><picture><source width="120" media="(prefers-color-scheme: dark)" srcset="https://docs.crawl4ai.com/uploads/sponsors/20251013045338_72a71fa4ee4d2f40.png"><source width="120" media="(prefers-color-scheme: light)" srcset="https://www.capsolver.com/assets/images/logo-text.png"><img alt="Capsolver" src="https://www.capsolver.com/assets/images/logo-text.png"></picture></a> | AI-powered Captcha solving service. Supports all major Captcha types, including reCAPTCHA, Cloudflare, and more | 🥈 Silver |
-| <a href="https://kipo.ai" target="_blank"><img src="https://docs.crawl4ai.com/uploads/sponsors/20251013045751_2d54f57f117c651e.png" alt="DataSync" width="120"/></a> | Helps engineers and buyers find, compare, and source electronic & industrial parts in seconds, with specs, pricing, lead times & alternatives.| 🥇 Gold |
-| <a href="https://www.kidocode.com/" target="_blank"><img src="https://docs.crawl4ai.com/uploads/sponsors/20251013045045_bb8dace3f0440d65.svg" alt="Kidocode" width="120"/><p align="center">KidoCode</p></a> | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold |
-| <a href="https://www.alephnull.sg/" target="_blank"><img src="https://docs.crawl4ai.com/uploads/sponsors/20251013050323_a9e8e8c4c3650421.svg" alt="Aleph null" width="120"/></a> | Singapore-based  Aleph Null is Asia’s leading edtech hub, dedicated to student-centric, AI-driven education—empowering learners with the tools to thrive in a fast-changing world. | 🥇 Gold |
-
-### 🧑‍🤝 Individual Sponsors
-
-A heartfelt thanks to our individual supporters! Every contribution helps us keep our opensource mission alive and thriving!
-
-<p align="left">
-  <a href="https://github.com/hafezparast"><img src="https://avatars.githubusercontent.com/u/14273305?s=60&v=4" style="border-radius:50%;" width="64px;"/></a>
-  <a href="https://github.com/ntohidi"><img src="https://avatars.githubusercontent.com/u/17140097?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
-  <a href="https://github.com/Sjoeborg"><img src="https://avatars.githubusercontent.com/u/17451310?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
-  <a href="https://github.com/romek-rozen"><img src="https://avatars.githubusercontent.com/u/30595969?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
-  <a href="https://github.com/Kourosh-Kiyani"><img src="https://avatars.githubusercontent.com/u/34105600?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
-  <a href="https://github.com/Etherdrake"><img src="https://avatars.githubusercontent.com/u/67021215?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
-  <a href="https://github.com/shaman247"><img src="https://avatars.githubusercontent.com/u/211010067?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
-  <a href="https://github.com/work-flow-manager"><img src="https://avatars.githubusercontent.com/u/217665461?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
-</p>
-
-> Want to join them? [Sponsor Crawl4AI →](https://github.com/sponsors/unclecode)
-
 ## Star History
 
 [![Star History Chart](https://api.star-history.com/svg?repos=unclecode/crawl4ai&type=Date)](https://star-history.com/#unclecode/crawl4ai&Date)

crawl4ai/__init__.py

@@ -103,8 +103,7 @@ from .browser_adapter import (
 
 from .utils import (
     start_colab_display_server,
-    setup_colab_environment,
+    setup_colab_environment
-    hooks_to_string
 )
 
 __all__ = [
@@ -184,7 +183,6 @@ __all__ = [
     "ProxyConfig",
     "start_colab_display_server",
     "setup_colab_environment",
-    "hooks_to_string",
     # C4A Script additions
     "c4a_compile",
     "c4a_validate", 

crawl4ai/__version__.py

@@ -1,7 +1,7 @@
 # crawl4ai/__version__.py
 
 # This is the version that will be used for stable releases
-__version__ = "0.7.6"
+__version__ = "0.7.4"
 
 # For nightly builds, this gets set during build process
 __nightly_version__ = None

crawl4ai/cli.py

@@ -2,8 +2,6 @@ import click
 import os
 import sys
 import time
-import subprocess
-import shutil
 
 import humanize
 from typing import Dict, Any, Optional, List
@@ -627,76 +625,6 @@ def cli():
     pass
 
 
-# Register server command group (Docker orchestration)
-# Redirect to standalone 'cnode' CLI
-@cli.command("server", context_settings=dict(
-    ignore_unknown_options=True,
-    allow_extra_args=True,
-    allow_interspersed_args=False
-))
-@click.pass_context
-def server_cmd(ctx):
-    """Manage Crawl4AI Docker server instances (deprecated - use 'cnode')
-
-    This command has been moved to a standalone CLI called 'cnode'.
-    For new installations, use:
-        curl -sSL https://crawl4ai.com/deploy.sh | bash
-
-    This redirect allows existing scripts to continue working.
-
-    Available commands: start, stop, status, scale, logs
-    Use 'crwl server <command> --help' for command-specific help.
-    """
-    # Check if cnode is installed
-    cnode_path = shutil.which("cnode")
-
-    # Get all the args (subcommand + options)
-    args = ctx.args
-
-    if not cnode_path:
-        console.print(Panel(
-            "[yellow]The 'crwl server' command has been moved to a standalone CLI.[/yellow]\n\n"
-            "Please install 'cnode' (Crawl4AI Node Manager):\n"
-            "[cyan]curl -sSL https://crawl4ai.com/deploy.sh | bash[/cyan]\n\n"
-            "After installation, use:\n"
-            "[green]cnode <command>[/green] instead of [dim]crwl server <command>[/dim]\n\n"
-            "For backward compatibility, we're using the local version for now.",
-            title="Server Command Moved",
-            border_style="yellow"
-        ))
-        # Try to use local version
-        try:
-            import sys
-            # Add deploy/docker to path
-            deploy_path = str(Path(__file__).parent.parent / 'deploy' / 'docker')
-            if deploy_path not in sys.path:
-                sys.path.insert(0, deploy_path)
-
-            from cnode_cli import cli as cnode_cli
-
-            # Forward to cnode with the args
-            sys.argv = ['cnode'] + args
-            cnode_cli(standalone_mode=False)
-            sys.exit(0)
-        except SystemExit as e:
-            # Normal exit from click
-            sys.exit(e.code if hasattr(e, 'code') else 0)
-        except Exception as e:
-            console.print(f"[red]Error: Could not find cnode or local server CLI: {e}[/red]")
-            console.print(f"[dim]Details: {e}[/dim]")
-            import traceback
-            console.print(f"[dim]{traceback.format_exc()}[/dim]")
-            sys.exit(1)
-
-    # cnode is installed - forward everything to it
-    try:
-        result = subprocess.run([cnode_path] + args, check=False)
-        sys.exit(result.returncode)
-    except Exception as e:
-        console.print(f"[red]Error running cnode: {e}[/red]")
-        sys.exit(1)
-
-
 @cli.group("browser")
 def browser_cmd():
     """Manage browser instances for Crawl4AI
@@ -1534,15 +1462,9 @@ def default(url: str, example: bool, browser_config: str, crawler_config: str, f
 
 def main():
     import sys
-    # Don't auto-insert 'crawl' if the command is recognized
+    if len(sys.argv) < 2 or sys.argv[1] not in cli.commands:
-    if len(sys.argv) >= 2 and sys.argv[1] in cli.commands:
-        cli()
-    elif len(sys.argv) < 2:
-        cli()
-    else:
-        # Unknown command - insert 'crawl' for backward compat
         sys.argv.insert(1, "crawl")
-        cli()
+    cli()
 
 if __name__ == "__main__":
     main()

crawl4ai/docker_client.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Union, AsyncGenerator, Dict, Any, Callable
+from typing import List, Optional, Union, AsyncGenerator, Dict, Any
 import httpx
 import json
 from urllib.parse import urljoin
@@ -7,7 +7,6 @@ import asyncio
 from .async_configs import BrowserConfig, CrawlerRunConfig
 from .models import CrawlResult
 from .async_logger import AsyncLogger, LogLevel
-from .utils import hooks_to_string
 
 
 class Crawl4aiClientError(Exception):
@@ -71,41 +70,17 @@ class Crawl4aiDockerClient:
             self.logger.error(f"Server unreachable: {str(e)}", tag="ERROR")
             raise ConnectionError(f"Cannot connect to server: {str(e)}")
 
-    def _prepare_request(
+    def _prepare_request(self, urls: List[str], browser_config: Optional[BrowserConfig] = None, 
-        self,
+                       crawler_config: Optional[CrawlerRunConfig] = None) -> Dict[str, Any]:
-        urls: List[str],
-        browser_config: Optional[BrowserConfig] = None,
-        crawler_config: Optional[CrawlerRunConfig] = None,
-        hooks: Optional[Union[Dict[str, Callable], Dict[str, str]]] = None,
-        hooks_timeout: int = 30
-    ) -> Dict[str, Any]:
         """Prepare request data from configs."""
         if self._token:
             self._http_client.headers["Authorization"] = f"Bearer {self._token}"
-
+        return {
-        request_data = {
             "urls": urls,
             "browser_config": browser_config.dump() if browser_config else {},
             "crawler_config": crawler_config.dump() if crawler_config else {}
         }
 
-        # Handle hooks if provided
-        if hooks:
-            # Check if hooks are already strings or need conversion
-            if any(callable(v) for v in hooks.values()):
-                # Convert function objects to strings
-                hooks_code = hooks_to_string(hooks)
-            else:
-                # Already in string format
-                hooks_code = hooks
-
-            request_data["hooks"] = {
-                "code": hooks_code,
-                "timeout": hooks_timeout
-            }
-
-        return request_data
-
     async def _request(self, method: str, endpoint: str, **kwargs) -> httpx.Response:
         """Make an HTTP request with error handling."""
         url = urljoin(self.base_url, endpoint)
@@ -127,42 +102,16 @@ class Crawl4aiDockerClient:
         self,
         urls: List[str],
         browser_config: Optional[BrowserConfig] = None,
-        crawler_config: Optional[CrawlerRunConfig] = None,
+        crawler_config: Optional[CrawlerRunConfig] = None
-        hooks: Optional[Union[Dict[str, Callable], Dict[str, str]]] = None,
-        hooks_timeout: int = 30
     ) -> Union[CrawlResult, List[CrawlResult], AsyncGenerator[CrawlResult, None]]:
-        """
+        """Execute a crawl operation."""
-        Execute a crawl operation.
-
-        Args:
-            urls: List of URLs to crawl
-            browser_config: Browser configuration
-            crawler_config: Crawler configuration
-            hooks: Optional hooks - can be either:
-                   - Dict[str, Callable]: Function objects that will be converted to strings
-                   - Dict[str, str]: Already stringified hook code
-            hooks_timeout: Timeout in seconds for each hook execution (1-120)
-
-        Returns:
-            Single CrawlResult, list of results, or async generator for streaming
-
-        Example with function hooks:
-            >>> async def my_hook(page, context, **kwargs):
-            ...     await page.set_viewport_size({"width": 1920, "height": 1080})
-            ...     return page
-            >>>
-            >>> result = await client.crawl(
-            ...     ["https://example.com"],
-            ...     hooks={"on_page_context_created": my_hook}
-            ... )
-        """
         await self._check_server()
-
+        
-        data = self._prepare_request(urls, browser_config, crawler_config, hooks, hooks_timeout)
+        data = self._prepare_request(urls, browser_config, crawler_config)
         is_streaming = crawler_config and crawler_config.stream
-
+        
         self.logger.info(f"Crawling {len(urls)} URLs {'(streaming)' if is_streaming else ''}", tag="CRAWL")
-
+        
         if is_streaming:
             async def stream_results() -> AsyncGenerator[CrawlResult, None]:
                 async with self._http_client.stream("POST", f"{self.base_url}/crawl/stream", json=data) as response:
@@ -179,12 +128,12 @@ class Crawl4aiDockerClient:
                             else:
                                 yield CrawlResult(**result)
             return stream_results()
-
+        
         response = await self._request("POST", "/crawl", json=data)
         result_data = response.json()
         if not result_data.get("success", False):
             raise RequestError(f"Crawl failed: {result_data.get('msg', 'Unknown error')}")
-
+        
         results = [CrawlResult(**r) for r in result_data.get("results", [])]
         self.logger.success(f"Crawl completed with {len(results)} results", tag="CRAWL")
         return results[0] if len(results) == 1 else results

crawl4ai/server_cli.py

@@ -1,479 +0,0 @@
-"""
-Crawl4AI Server CLI Commands
-
-Provides `crwl server` command group for Docker orchestration.
-"""
-
-import click
-import anyio
-from rich.console import Console
-from rich.table import Table
-from rich.panel import Panel
-from rich.prompt import Confirm
-
-from crawl4ai.server_manager import ServerManager
-
-
-console = Console()
-
-
-@click.group("server")
-def server_cmd():
-    """Manage Crawl4AI Docker server instances
-
-    One-command deployment with automatic scaling:
-    - Single container for development (N=1)
-    - Docker Swarm for production with built-in load balancing (N>1)
-    - Docker Compose + Nginx as fallback (N>1)
-
-    Examples:
-        crwl server start                    # Single container on port 11235
-        crwl server start --replicas 3       # Auto-detect Swarm or Compose
-        crwl server start -r 5 --port 8080   # 5 replicas on custom port
-        crwl server status                   # Check current deployment
-        crwl server scale 10                 # Scale to 10 replicas
-        crwl server stop                     # Stop and cleanup
-    """
-    pass
-
-
-@server_cmd.command("start")
-@click.option(
-    "--replicas", "-r",
-    type=int,
-    default=1,
-    help="Number of container replicas (default: 1)"
-)
-@click.option(
-    "--mode",
-    type=click.Choice(["auto", "single", "swarm", "compose"]),
-    default="auto",
-    help="Deployment mode (default: auto-detect)"
-)
-@click.option(
-    "--port", "-p",
-    type=int,
-    default=11235,
-    help="External port to expose (default: 11235)"
-)
-@click.option(
-    "--env-file",
-    type=click.Path(exists=True),
-    help="Path to environment file"
-)
-@click.option(
-    "--image",
-    default="unclecode/crawl4ai:latest",
-    help="Docker image to use (default: unclecode/crawl4ai:latest)"
-)
-def start_cmd(replicas: int, mode: str, port: int, env_file: str, image: str):
-    """Start Crawl4AI server with automatic orchestration.
-
-    Deployment modes:
-    - auto: Automatically choose best mode (default)
-    - single: Single container (N=1 only)
-    - swarm: Docker Swarm with built-in load balancing
-    - compose: Docker Compose + Nginx reverse proxy
-
-    The server will:
-    1. Check if Docker is running
-    2. Validate port availability
-    3. Pull image if needed
-    4. Start container(s) with health checks
-    5. Save state for management
-
-    Examples:
-        # Development: single container
-        crwl server start
-
-        # Production: 5 replicas with Swarm
-        crwl server start --replicas 5
-
-        # Custom configuration
-        crwl server start -r 3 --port 8080 --env-file .env.prod
-    """
-    manager = ServerManager()
-
-    console.print(Panel(
-        f"[cyan]Starting Crawl4AI Server[/cyan]\n\n"
-        f"Replicas: [yellow]{replicas}[/yellow]\n"
-        f"Mode: [yellow]{mode}[/yellow]\n"
-        f"Port: [yellow]{port}[/yellow]\n"
-        f"Image: [yellow]{image}[/yellow]",
-        title="Server Start",
-        border_style="cyan"
-    ))
-
-    with console.status("[cyan]Starting server..."):
-        async def _start():
-            return await manager.start(
-                replicas=replicas,
-                mode=mode,
-                port=port,
-                env_file=env_file,
-                image=image
-            )
-        result = anyio.run(_start)
-
-    if result["success"]:
-        console.print(Panel(
-            f"[green]✓ Server started successfully![/green]\n\n"
-            f"Mode: [cyan]{result.get('state_data', {}).get('mode', mode)}[/cyan]\n"
-            f"URL: [bold]http://localhost:{port}[/bold]\n"
-            f"Health: [bold]http://localhost:{port}/health[/bold]\n"
-            f"Monitor: [bold]http://localhost:{port}/monitor[/bold]",
-            title="Server Running",
-            border_style="green"
-        ))
-    else:
-        error_msg = result.get("error", result.get("message", "Unknown error"))
-        console.print(Panel(
-            f"[red]✗ Failed to start server[/red]\n\n"
-            f"{error_msg}",
-            title="Error",
-            border_style="red"
-        ))
-
-        if "already running" in error_msg.lower():
-            console.print("\n[yellow]Hint: Use 'crwl server status' to check current deployment[/yellow]")
-            console.print("[yellow]      Use 'crwl server stop' to stop existing server[/yellow]")
-
-
-@server_cmd.command("status")
-def status_cmd():
-    """Show current server status and deployment info.
-
-    Displays:
-    - Running state (up/down)
-    - Deployment mode (single/swarm/compose)
-    - Number of replicas
-    - Port mapping
-    - Uptime
-    - Image version
-
-    Example:
-        crwl server status
-    """
-    manager = ServerManager()
-
-    async def _status():
-        return await manager.status()
-    result = anyio.run(_status)
-
-    if result["running"]:
-        table = Table(title="Crawl4AI Server Status", border_style="green")
-        table.add_column("Property", style="cyan")
-        table.add_column("Value", style="green")
-
-        table.add_row("Status", "🟢 Running")
-        table.add_row("Mode", result["mode"])
-        table.add_row("Replicas", str(result.get("replicas", 1)))
-        table.add_row("Port", str(result.get("port", 11235)))
-        table.add_row("Image", result.get("image", "unknown"))
-        table.add_row("Uptime", result.get("uptime", "unknown"))
-        table.add_row("Started", result.get("started_at", "unknown"))
-
-        console.print(table)
-        console.print(f"\n[green]✓ Server is healthy[/green]")
-        console.print(f"[dim]Access: http://localhost:{result.get('port', 11235)}[/dim]")
-    else:
-        console.print(Panel(
-            f"[yellow]No server is currently running[/yellow]\n\n"
-            f"Use 'crwl server start' to launch a server",
-            title="Server Status",
-            border_style="yellow"
-        ))
-
-
-@server_cmd.command("stop")
-@click.option(
-    "--remove-volumes",
-    is_flag=True,
-    help="Remove associated volumes (WARNING: deletes data)"
-)
-def stop_cmd(remove_volumes: bool):
-    """Stop running Crawl4AI server and cleanup resources.
-
-    This will:
-    1. Stop all running containers/services
-    2. Remove containers
-    3. Optionally remove volumes (--remove-volumes)
-    4. Clean up state files
-
-    WARNING: Use --remove-volumes with caution as it will delete
-    persistent data including Redis databases and logs.
-
-    Examples:
-        # Stop server, keep volumes
-        crwl server stop
-
-        # Stop and remove all data
-        crwl server stop --remove-volumes
-    """
-    manager = ServerManager()
-
-    # Confirm if removing volumes
-    if remove_volumes:
-        if not Confirm.ask(
-            "[red]⚠️  This will delete all server data including Redis databases. Continue?[/red]"
-        ):
-            console.print("[yellow]Cancelled[/yellow]")
-            return
-
-    with console.status("[cyan]Stopping server..."):
-        async def _stop():
-            return await manager.stop(remove_volumes=remove_volumes)
-        result = anyio.run(_stop)
-
-    if result["success"]:
-        console.print(Panel(
-            f"[green]✓ Server stopped successfully[/green]\n\n"
-            f"{result.get('message', 'All resources cleaned up')}",
-            title="Server Stopped",
-            border_style="green"
-        ))
-    else:
-        console.print(Panel(
-            f"[red]✗ Error stopping server[/red]\n\n"
-            f"{result.get('error', result.get('message', 'Unknown error'))}",
-            title="Error",
-            border_style="red"
-        ))
-
-
-@server_cmd.command("scale")
-@click.argument("replicas", type=int)
-def scale_cmd(replicas: int):
-    """Scale server to specified number of replicas.
-
-    Only works with Swarm or Compose modes. Single container
-    mode cannot be scaled (must stop and restart with --replicas).
-
-    Scaling is live and does not require downtime. The load
-    balancer will automatically distribute traffic to new replicas.
-
-    Examples:
-        # Scale up to 10 replicas
-        crwl server scale 10
-
-        # Scale down to 2 replicas
-        crwl server scale 2
-
-        # Scale to 1 (minimum)
-        crwl server scale 1
-    """
-    if replicas < 1:
-        console.print("[red]Error: Replicas must be at least 1[/red]")
-        return
-
-    manager = ServerManager()
-
-    with console.status(f"[cyan]Scaling to {replicas} replicas..."):
-        async def _scale():
-            return await manager.scale(replicas=replicas)
-        result = anyio.run(_scale)
-
-    if result["success"]:
-        console.print(Panel(
-            f"[green]✓ Scaled successfully[/green]\n\n"
-            f"New replica count: [bold]{replicas}[/bold]\n"
-            f"Mode: [cyan]{result.get('mode')}[/cyan]",
-            title="Scaling Complete",
-            border_style="green"
-        ))
-    else:
-        error_msg = result.get("error", result.get("message", "Unknown error"))
-        console.print(Panel(
-            f"[red]✗ Scaling failed[/red]\n\n"
-            f"{error_msg}",
-            title="Error",
-            border_style="red"
-        ))
-
-        if "single container" in error_msg.lower():
-            console.print("\n[yellow]Hint: For single container mode:[/yellow]")
-            console.print("[yellow]  1. crwl server stop[/yellow]")
-            console.print(f"[yellow]  2. crwl server start --replicas {replicas}[/yellow]")
-
-
-@server_cmd.command("logs")
-@click.option(
-    "--follow", "-f",
-    is_flag=True,
-    help="Follow log output (like tail -f)"
-)
-@click.option(
-    "--tail",
-    type=int,
-    default=100,
-    help="Number of lines to show (default: 100)"
-)
-def logs_cmd(follow: bool, tail: int):
-    """View server logs.
-
-    Shows logs from running containers/services. Use --follow
-    to stream logs in real-time.
-
-    Examples:
-        # Show last 100 lines
-        crwl server logs
-
-        # Show last 500 lines
-        crwl server logs --tail 500
-
-        # Follow logs in real-time
-        crwl server logs --follow
-
-        # Combine options
-        crwl server logs -f --tail 50
-    """
-    manager = ServerManager()
-
-    async def _logs():
-        return await manager.logs(follow=follow, tail=tail)
-    output = anyio.run(_logs)
-    console.print(output)
-
-
-@server_cmd.command("cleanup")
-@click.option(
-    "--force",
-    is_flag=True,
-    help="Force cleanup even if state file doesn't exist"
-)
-def cleanup_cmd(force: bool):
-    """Force cleanup of all Crawl4AI Docker resources.
-
-    Stops and removes all containers, networks, and optionally volumes.
-    Useful when server is stuck or state is corrupted.
-
-    Examples:
-        # Clean up everything
-        crwl server cleanup
-
-        # Force cleanup (ignore state file)
-        crwl server cleanup --force
-    """
-    manager = ServerManager()
-
-    console.print(Panel(
-        f"[yellow]⚠️  Cleaning up Crawl4AI Docker resources[/yellow]\n\n"
-        f"This will stop and remove:\n"
-        f"- All Crawl4AI containers\n"
-        f"- Nginx load balancer\n"
-        f"- Redis instance\n"
-        f"- Docker networks\n"
-        f"- State files",
-        title="Cleanup",
-        border_style="yellow"
-    ))
-
-    if not force and not Confirm.ask("[yellow]Continue with cleanup?[/yellow]"):
-        console.print("[yellow]Cancelled[/yellow]")
-        return
-
-    with console.status("[cyan]Cleaning up resources..."):
-        async def _cleanup():
-            return await manager.cleanup(force=force)
-        result = anyio.run(_cleanup)
-
-    if result["success"]:
-        console.print(Panel(
-            f"[green]✓ Cleanup completed successfully[/green]\n\n"
-            f"Removed: {result.get('removed', 0)} containers\n"
-            f"{result.get('message', 'All resources cleaned up')}",
-            title="Cleanup Complete",
-            border_style="green"
-        ))
-    else:
-        console.print(Panel(
-            f"[yellow]⚠️  Partial cleanup[/yellow]\n\n"
-            f"{result.get('message', 'Some resources may still exist')}",
-            title="Cleanup Status",
-            border_style="yellow"
-        ))
-
-
-@server_cmd.command("restart")
-@click.option(
-    "--replicas", "-r",
-    type=int,
-    help="New replica count (optional)"
-)
-def restart_cmd(replicas: int):
-    """Restart server (stop then start with same config).
-
-    Preserves existing configuration unless overridden with options.
-    Useful for applying image updates or recovering from errors.
-
-    Examples:
-        # Restart with same configuration
-        crwl server restart
-
-        # Restart and change replica count
-        crwl server restart --replicas 5
-    """
-    manager = ServerManager()
-
-    # Get current state
-    async def _get_status():
-        return await manager.status()
-    current = anyio.run(_get_status)
-
-    if not current["running"]:
-        console.print("[yellow]No server is running. Use 'crwl server start' instead.[/yellow]")
-        return
-
-    # Extract current config
-    current_replicas = current.get("replicas", 1)
-    current_port = current.get("port", 11235)
-    current_image = current.get("image", "unclecode/crawl4ai:latest")
-    current_mode = current.get("mode", "auto")
-
-    # Override with CLI args
-    new_replicas = replicas if replicas is not None else current_replicas
-
-    console.print(Panel(
-        f"[cyan]Restarting Crawl4AI Server[/cyan]\n\n"
-        f"Replicas: [yellow]{current_replicas}[/yellow] → [green]{new_replicas}[/green]\n"
-        f"Port: [yellow]{current_port}[/yellow]\n"
-        f"Mode: [yellow]{current_mode}[/yellow]",
-        title="Server Restart",
-        border_style="cyan"
-    ))
-
-    # Stop current
-    with console.status("[cyan]Stopping current server..."):
-        async def _stop_server():
-            return await manager.stop(remove_volumes=False)
-        stop_result = anyio.run(_stop_server)
-
-    if not stop_result["success"]:
-        console.print(f"[red]Failed to stop server: {stop_result.get('error')}[/red]")
-        return
-
-    # Start new
-    with console.status("[cyan]Starting server..."):
-        async def _start_server():
-            return await manager.start(
-                replicas=new_replicas,
-                mode="auto",
-                port=current_port,
-                image=current_image
-            )
-        start_result = anyio.run(_start_server)
-
-    if start_result["success"]:
-        console.print(Panel(
-            f"[green]✓ Server restarted successfully![/green]\n\n"
-            f"URL: [bold]http://localhost:{current_port}[/bold]",
-            title="Restart Complete",
-            border_style="green"
-        ))
-    else:
-        console.print(Panel(
-            f"[red]✗ Failed to restart server[/red]\n\n"
-            f"{start_result.get('error', 'Unknown error')}",
-            title="Error",
-            border_style="red"
-        ))

crawl4ai/templates/docker-compose.template.yml

@@ -1,52 +0,0 @@
-version: '3.8'
-
-services:
-  redis:
-    image: redis:alpine
-    command: redis-server --appendonly yes
-    volumes:
-      - redis_data:/data
-    networks:
-      - crawl4ai_net
-    restart: unless-stopped
-
-  crawl4ai:
-    image: ${IMAGE}
-    deploy:
-      replicas: ${REPLICAS}
-      resources:
-        limits:
-          memory: 4G
-    shm_size: 1g
-    environment:
-      - REDIS_HOST=redis
-      - REDIS_PORT=6379
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:11235/health"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-      start_period: 40s
-    depends_on:
-      - redis
-    networks:
-      - crawl4ai_net
-
-  nginx:
-    image: nginx:alpine
-    ports:
-      - "${PORT}:80"
-    volumes:
-      - ${NGINX_CONF}:/etc/nginx/nginx.conf:ro
-    depends_on:
-      - crawl4ai
-    networks:
-      - crawl4ai_net
-    restart: unless-stopped
-
-networks:
-  crawl4ai_net:
-    driver: bridge
-
-volumes:
-  redis_data:

crawl4ai/templates/nginx.conf.template

@@ -1,75 +0,0 @@
-events {
-    worker_connections 1024;
-}
-
-http {
-    upstream crawl4ai_backend {
-        # DNS-based load balancing to Docker Compose service
-        # Docker Compose provides DNS resolution for service name
-        server crawl4ai:11235 max_fails=3 fail_timeout=30s;
-
-        # Keep connections alive
-        keepalive 32;
-    }
-
-    # Sticky sessions for monitoring (same IP always goes to same container)
-    upstream crawl4ai_monitor {
-        ip_hash;  # Sticky sessions based on client IP
-        server crawl4ai:11235 max_fails=3 fail_timeout=30s;
-        keepalive 32;
-    }
-
-    server {
-        listen 80;
-        server_name _;
-
-        # Increase timeouts for long-running crawl operations
-        proxy_connect_timeout 300;
-        proxy_send_timeout 300;
-        proxy_read_timeout 300;
-        send_timeout 300;
-
-        # WebSocket endpoint for real-time monitoring (exact match)
-        location = /monitor/ws {
-            proxy_pass http://crawl4ai_monitor/monitor/ws;
-            proxy_http_version 1.1;
-            proxy_set_header Upgrade $http_upgrade;
-            proxy_set_header Connection "upgrade";
-            proxy_set_header Host $host;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-
-            # WebSocket timeouts
-            proxy_connect_timeout 7d;
-            proxy_send_timeout 7d;
-            proxy_read_timeout 7d;
-        }
-
-        # Monitor and dashboard with sticky sessions (regex location)
-        location ~ ^/(monitor|dashboard) {
-            proxy_pass http://crawl4ai_monitor;
-            proxy_set_header Host $host;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-Forwarded-Proto $scheme;
-        }
-
-        # HTTP endpoints (load balanced)
-        location / {
-            proxy_pass http://crawl4ai_backend;
-            proxy_set_header Host $host;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-Forwarded-Proto $scheme;
-
-            # Support large request bodies (for batch operations)
-            client_max_body_size 10M;
-        }
-
-        # Health check endpoint (bypass load balancer)
-        location /health {
-            proxy_pass http://crawl4ai_backend/health;
-            access_log off;
-        }
-    }
-}

crawl4ai/utils.py

@@ -47,7 +47,6 @@ from urllib.parse import (
     urljoin, urlparse, urlunparse,
     parse_qsl, urlencode, quote, unquote
 )
-import inspect
 
 
 # Monkey patch to fix wildcard handling in urllib.robotparser
@@ -3530,52 +3529,4 @@ def get_memory_stats() -> Tuple[float, float, float]:
     available_gb = get_true_available_memory_gb()
     used_percent = get_true_memory_usage_percent()
     
-    return used_percent, available_gb, total_gb
+    return used_percent, available_gb, total_gb
-
-
-# Hook utilities for Docker API
-def hooks_to_string(hooks: Dict[str, Callable]) -> Dict[str, str]:
-    """
-    Convert hook function objects to string representations for Docker API.
-
-    This utility simplifies the process of using hooks with the Docker API by converting
-    Python function objects into the string format required by the API.
-
-    Args:
-        hooks: Dictionary mapping hook point names to Python function objects.
-               Functions should be async and follow hook signature requirements.
-
-    Returns:
-        Dictionary mapping hook point names to string representations of the functions.
-
-    Example:
-        >>> async def my_hook(page, context, **kwargs):
-        ...     await page.set_viewport_size({"width": 1920, "height": 1080})
-        ...     return page
-        >>>
-        >>> hooks_dict = {"on_page_context_created": my_hook}
-        >>> api_hooks = hooks_to_string(hooks_dict)
-        >>> # api_hooks is now ready to use with Docker API
-
-    Raises:
-        ValueError: If a hook is not callable or source cannot be extracted
-    """
-    result = {}
-
-    for hook_name, hook_func in hooks.items():
-        if not callable(hook_func):
-            raise ValueError(f"Hook '{hook_name}' must be a callable function, got {type(hook_func)}")
-
-        try:
-            # Get the source code of the function
-            source = inspect.getsource(hook_func)
-            # Remove any leading indentation to get clean source
-            source = textwrap.dedent(source)
-            result[hook_name] = source
-        except (OSError, TypeError) as e:
-            raise ValueError(
-                f"Cannot extract source code for hook '{hook_name}'. "
-                f"Make sure the function is defined in a file (not interactively). Error: {e}"
-            )
-
-    return result

deploy/docker/AGENT.md

@@ -1,402 +0,0 @@
-# Crawl4AI DevOps Agent Context
-
-## Service Overview
-**Crawl4AI**: Browser-based web crawling service with AI extraction. Docker deployment with horizontal scaling (1-N containers), Redis coordination, Nginx load balancing.
-
-## Architecture Quick Reference
-
-```
-Client → Nginx:11235 → [crawl4ai-1, crawl4ai-2, ...crawl4ai-N] ← Redis
-                              ↓
-                         Monitor Dashboard
-```
-
-**Components:**
-- **Nginx**: Load balancer (round-robin API, sticky monitoring)
-- **Crawl4AI containers**: FastAPI + Playwright browsers
-- **Redis**: Container discovery (heartbeats 30s), monitoring data aggregation
-- **Monitor**: Real-time dashboard at `/dashboard`
-
-## CLI Commands
-
-### Start/Stop
-```bash
-crwl server start [-r N] [--port P] [--mode auto|single|swarm|compose] [--env-file F] [--image I]
-crwl server stop [--remove-volumes]
-crwl server restart [-r N]
-```
-
-### Management
-```bash
-crwl server status        # Show mode, replicas, port, uptime
-crwl server scale N       # Live scaling (Swarm/Compose only)
-crwl server logs [-f] [--tail N]
-```
-
-**Defaults**: replicas=1, port=11235, mode=auto, image=unclecode/crawl4ai:latest
-
-## Deployment Modes
-
-| Replicas | Mode | Load Balancer | Use Case |
-|----------|------|---------------|----------|
-| N=1 | single | None | Dev/testing |
-| N>1 | swarm | Built-in | Production (if `docker swarm init` done) |
-| N>1 | compose | Nginx | Production (fallback) |
-
-**Mode Detection** (when mode=auto):
-1. If N=1 → single
-2. If N>1 & Swarm active → swarm
-3. If N>1 & Swarm inactive → compose
-
-## File Locations
-
-```
-~/.crawl4ai/server/
-├── state.json              # Current deployment state
-├── docker-compose.yml      # Generated compose file
-└── nginx.conf              # Generated nginx config
-
-/app/                       # Inside container
-├── deploy/docker/server.py
-├── deploy/docker/monitor.py
-├── deploy/docker/static/monitor/index.html
-└── crawler_pool.py         # Browser pool (PERMANENT, HOT_POOL, COLD_POOL)
-```
-
-## Monitoring & Troubleshooting
-
-### Health Checks
-```bash
-curl http://localhost:11235/health              # Service health
-curl http://localhost:11235/monitor/containers  # Container discovery
-curl http://localhost:11235/monitor/requests    # Aggregated requests
-```
-
-### Dashboard
-- URL: `http://localhost:11235/dashboard/`
-- Features: Container filtering (All/C-1/C-2/C-3), real-time WebSocket, timeline charts
-- WebSocket: `/monitor/ws` (sticky sessions)
-
-### Common Issues
-
-**No containers showing in dashboard:**
-```bash
-docker exec <redis-container> redis-cli SMEMBERS monitor:active_containers
-docker exec <redis-container> redis-cli KEYS "monitor:heartbeat:*"
-```
-Wait 30s for heartbeat registration.
-
-**Load balancing not working:**
-```bash
-docker exec <nginx-container> cat /etc/nginx/nginx.conf | grep upstream
-docker logs <nginx-container> | grep error
-```
-Check Nginx upstream has no `ip_hash` for API endpoints.
-
-**Redis connection errors:**
-```bash
-docker logs <crawl4ai-container> | grep -i redis
-docker exec <crawl4ai-container> ping redis
-```
-Verify REDIS_HOST=redis, REDIS_PORT=6379.
-
-**Containers not scaling:**
-```bash
-# Swarm
-docker service ls
-docker service ps crawl4ai
-
-# Compose
-docker compose -f ~/.crawl4ai/server/docker-compose.yml ps
-docker compose -f ~/.crawl4ai/server/docker-compose.yml up -d --scale crawl4ai=N
-```
-
-### Redis Data Structure
-```
-monitor:active_containers              # SET: {container_ids}
-monitor:heartbeat:{cid}                # STRING: {id, hostname, last_seen} TTL=60s
-monitor:{cid}:active_requests          # STRING: JSON list, TTL=5min
-monitor:{cid}:completed                # STRING: JSON list, TTL=1h
-monitor:{cid}:janitor                  # STRING: JSON list, TTL=1h
-monitor:{cid}:errors                   # STRING: JSON list, TTL=1h
-monitor:endpoint_stats                 # STRING: JSON aggregate, TTL=24h
-```
-
-## Environment Variables
-
-### Required for Multi-LLM
-```bash
-OPENAI_API_KEY=sk-...
-ANTHROPIC_API_KEY=sk-ant-...
-DEEPSEEK_API_KEY=...
-GROQ_API_KEY=...
-TOGETHER_API_KEY=...
-MISTRAL_API_KEY=...
-GEMINI_API_TOKEN=...
-```
-
-### Redis Configuration (Optional)
-```bash
-REDIS_HOST=redis                       # Default: redis
-REDIS_PORT=6379                        # Default: 6379
-REDIS_TTL_ACTIVE_REQUESTS=300          # Default: 5min
-REDIS_TTL_COMPLETED_REQUESTS=3600      # Default: 1h
-REDIS_TTL_JANITOR_EVENTS=3600          # Default: 1h
-REDIS_TTL_ERRORS=3600                  # Default: 1h
-REDIS_TTL_ENDPOINT_STATS=86400         # Default: 24h
-REDIS_TTL_HEARTBEAT=60                 # Default: 1min
-```
-
-## API Endpoints
-
-### Core API
-- `POST /crawl` - Crawl URL (load-balanced)
-- `POST /batch` - Batch crawl (load-balanced)
-- `GET /health` - Health check (load-balanced)
-
-### Monitor API (Aggregated from all containers)
-- `GET /monitor/health` - Local container health
-- `GET /monitor/containers` - All active containers
-- `GET /monitor/requests` - All requests (active + completed)
-- `GET /monitor/browsers` - Browser pool status (local only)
-- `GET /monitor/logs/janitor` - Janitor cleanup events
-- `GET /monitor/logs/errors` - Error logs
-- `GET /monitor/endpoints/stats` - Endpoint analytics
-- `WS /monitor/ws` - Real-time updates (aggregated)
-
-### Control Actions
-- `POST /monitor/actions/cleanup` - Force browser cleanup
-- `POST /monitor/actions/kill_browser` - Kill specific browser
-- `POST /monitor/actions/restart_browser` - Restart browser
-- `POST /monitor/stats/reset` - Reset endpoint counters
-
-## Docker Commands Reference
-
-### Inspection
-```bash
-# List containers
-docker ps --filter "name=crawl4ai"
-
-# Container logs
-docker logs <container-id> -f --tail 100
-
-# Redis CLI
-docker exec -it <redis-container> redis-cli
-KEYS monitor:*
-SMEMBERS monitor:active_containers
-GET monitor:<cid>:completed
-TTL monitor:heartbeat:<cid>
-
-# Nginx config
-docker exec <nginx-container> cat /etc/nginx/nginx.conf
-
-# Container stats
-docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"
-```
-
-### Compose Operations
-```bash
-# Scale
-docker compose -f ~/.crawl4ai/server/docker-compose.yml up -d --scale crawl4ai=5
-
-# Restart service
-docker compose -f ~/.crawl4ai/server/docker-compose.yml restart crawl4ai
-
-# View services
-docker compose -f ~/.crawl4ai/server/docker-compose.yml ps
-```
-
-### Swarm Operations
-```bash
-# Initialize Swarm
-docker swarm init
-
-# Scale service
-docker service scale crawl4ai=5
-
-# Service info
-docker service ls
-docker service ps crawl4ai --no-trunc
-
-# Service logs
-docker service logs crawl4ai --tail 100 -f
-```
-
-## Performance & Scaling
-
-### Resource Recommendations
-| Containers | Memory/Container | Total Memory | Use Case |
-|------------|-----------------|--------------|----------|
-| 1 | 4GB | 4GB | Development |
-| 3 | 4GB | 12GB | Small prod |
-| 5 | 4GB | 20GB | Medium prod |
-| 10 | 4GB | 40GB | Large prod |
-
-**Expected Throughput**: ~10 req/min per container (depends on crawl complexity)
-
-### Scaling Guidelines
-- **Horizontal**: Add replicas (`crwl server scale N`)
-- **Vertical**: Adjust `--memory 8G --cpus 4` in kwargs
-- **Browser Pool**: Permanent (1) + Hot pool (adaptive) + Cold pool (cleanup by janitor)
-
-### Redis Memory Usage
-- **Per container**: ~110KB (requests + events + errors + heartbeat)
-- **10 containers**: ~1.1MB
-- **Recommendation**: 256MB Redis is sufficient for <100 containers
-
-## Security Notes
-
-### Input Validation
-All CLI inputs validated:
-- Image name: alphanumeric + `.-/:_@` only, max 256 chars
-- Port: 1-65535
-- Replicas: 1-100
-- Env file: must exist and be readable
-- Container IDs: alphanumeric + `-_` only (prevents Redis injection)
-
-### Network Security
-- Nginx forwards to internal `crawl4ai` service (Docker network)
-- Monitor endpoints have NO authentication (add MONITOR_TOKEN env for security)
-- Redis is internal-only (no external port)
-
-### Recommended Production Setup
-```bash
-# Add authentication
-export MONITOR_TOKEN="your-secret-token"
-
-# Use Redis password
-redis:
-  command: redis-server --requirepass ${REDIS_PASSWORD}
-
-# Enable rate limiting in Nginx
-limit_req_zone $binary_remote_addr zone=api:10m rate=10r/s;
-```
-
-## Common User Scenarios
-
-### Scenario 1: Fresh Deployment
-```bash
-crwl server start --replicas 3 --env-file .env
-# Wait for health check, then access http://localhost:11235/health
-```
-
-### Scenario 2: Scaling Under Load
-```bash
-crwl server scale 10
-# Live scaling, no downtime
-```
-
-### Scenario 3: Debugging Slow Requests
-```bash
-# Check dashboard
-open http://localhost:11235/dashboard/
-
-# Check container logs
-docker logs <slowest-container-id> --tail 100
-
-# Check browser pool
-curl http://localhost:11235/monitor/browsers | jq
-```
-
-### Scenario 4: Redis Connection Issues
-```bash
-# Check Redis connectivity
-docker exec <crawl4ai-container> nc -zv redis 6379
-
-# Check Redis logs
-docker logs <redis-container>
-
-# Restart containers (triggers reconnect with retry logic)
-crwl server restart
-```
-
-### Scenario 5: Container Not Appearing in Dashboard
-```bash
-# Wait 30s for heartbeat
-sleep 30
-
-# Check Redis
-docker exec <redis-container> redis-cli SMEMBERS monitor:active_containers
-
-# Check container logs for heartbeat errors
-docker logs <missing-container> | grep -i heartbeat
-```
-
-## Code Context for Advanced Debugging
-
-### Key Classes
-- `MonitorStats` (monitor.py): Tracks stats, Redis persistence, heartbeat worker
-- `ServerManager` (server_manager.py): CLI orchestration, mode detection
-- Browser pool globals: `PERMANENT`, `HOT_POOL`, `COLD_POOL`, `LOCK` (crawler_pool.py)
-
-### Critical Timeouts
-- Browser pool lock: 2s timeout (prevents deadlock)
-- WebSocket connection: 5s timeout
-- Health check: 30-60s timeout
-- Heartbeat interval: 30s, TTL: 60s
-- Redis retry: 3 attempts, backoff: 0.5s/1s/2s
-- Circuit breaker: 5 failures → 5min backoff
-
-### State Transitions
-```
-NOT_RUNNING → STARTING → HEALTHY → RUNNING
-                ↓           ↓
-            FAILED      UNHEALTHY → STOPPED
-```
-
-State file: `~/.crawl4ai/server/state.json` (atomic writes, fcntl locking)
-
-## Quick Diagnostic Commands
-
-```bash
-# Full system check
-crwl server status
-docker ps
-curl http://localhost:11235/health
-curl http://localhost:11235/monitor/containers | jq
-
-# Redis check
-docker exec <redis-container> redis-cli PING
-docker exec <redis-container> redis-cli INFO stats
-
-# Network check
-docker network ls
-docker network inspect <network-name>
-
-# Logs check
-docker logs <nginx-container> --tail 50
-docker logs <redis-container> --tail 50
-docker compose -f ~/.crawl4ai/server/docker-compose.yml logs --tail 100
-```
-
-## Agent Decision Tree
-
-**User reports slow crawling:**
-1. Check dashboard for active requests stuck → kill browser if >5min
-2. Check browser pool status → cleanup if hot/cold pool >10
-3. Check container CPU/memory → scale up if >80%
-4. Check Redis latency → restart Redis if >100ms
-
-**User reports missing containers:**
-1. Wait 30s for heartbeat
-2. Check `docker ps` vs dashboard count
-3. Check Redis SMEMBERS monitor:active_containers
-4. Check container logs for Redis connection errors
-5. Verify REDIS_HOST/PORT env vars
-
-**User reports 502/503 errors:**
-1. Check Nginx logs for upstream errors
-2. Check container health: `curl http://localhost:11235/health`
-3. Check if all containers are healthy: `docker ps`
-4. Restart Nginx: `docker restart <nginx-container>`
-
-**User wants to update image:**
-1. `crwl server stop`
-2. `docker pull unclecode/crawl4ai:latest`
-3. `crwl server start --replicas <previous-count>`
-
----
-
-**Version**: Crawl4AI v0.7.4+
-**Last Updated**: 2025-01-20
-**AI Agent Note**: All commands, file paths, and Redis keys verified against codebase. Use exact syntax shown. For user-facing responses, translate technical details to plain language.

deploy/docker/README.md

@@ -12,7 +12,6 @@
   - [Python SDK](#python-sdk)
   - [Understanding Request Schema](#understanding-request-schema)
   - [REST API Examples](#rest-api-examples)
-  - [Asynchronous Jobs with Webhooks](#asynchronous-jobs-with-webhooks)
 - [Additional API Endpoints](#additional-api-endpoints)
   - [HTML Extraction Endpoint](#html-extraction-endpoint)
   - [Screenshot Endpoint](#screenshot-endpoint)
@@ -59,13 +58,15 @@ Pull and run images directly from Docker Hub without building locally.
 
 #### 1. Pull the Image
 
-Our latest stable release is `0.7.6`. Images are built with multi-arch manifests, so Docker automatically pulls the correct version for your system.
+Our latest release candidate is `0.7.0-r1`. Images are built with multi-arch manifests, so Docker automatically pulls the correct version for your system.
+
+> ⚠️ **Important Note**: The `latest` tag currently points to the stable `0.6.0` version. After testing and validation, `0.7.0` (without -r1) will be released and `latest` will be updated. For now, please use `0.7.0-r1` to test the new features.
 
 ```bash
-# Pull the latest stable version (0.7.6)
+# Pull the release candidate (for testing new features)
-docker pull unclecode/crawl4ai:0.7.6
+docker pull unclecode/crawl4ai:0.7.0-r1
 
-# Or use the latest tag (points to 0.7.6)
+# Or pull the current stable version (0.6.0)
 docker pull unclecode/crawl4ai:latest
 ```
 
@@ -100,7 +101,7 @@ EOL
       -p 11235:11235 \
       --name crawl4ai \
       --shm-size=1g \
-      unclecode/crawl4ai:0.7.6
+      unclecode/crawl4ai:0.7.0-r1
     ```
 
 *   **With LLM support:**
@@ -111,7 +112,7 @@ EOL
       --name crawl4ai \
       --env-file .llm.env \
       --shm-size=1g \
-      unclecode/crawl4ai:0.7.6
+      unclecode/crawl4ai:0.7.0-r1
     ```
 
 > The server will be available at `http://localhost:11235`. Visit `/playground` to access the interactive testing interface.
@@ -184,7 +185,7 @@ The `docker-compose.yml` file in the project root provides a simplified approach
     ```bash
     # Pulls and runs the release candidate from Docker Hub
     # Automatically selects the correct architecture
-    IMAGE=unclecode/crawl4ai:0.7.6 docker compose up -d
+    IMAGE=unclecode/crawl4ai:0.7.0-r1 docker compose up -d
     ```
 
 *   **Build and Run Locally:**
@@ -647,194 +648,6 @@ async def test_stream_crawl(token: str = None): # Made token optional
 # asyncio.run(test_stream_crawl())
 ```
 
-### Asynchronous Jobs with Webhooks
-
-For long-running crawls or when you want to avoid keeping connections open, use the job queue endpoints. Instead of polling for results, configure a webhook to receive notifications when jobs complete.
-
-#### Why Use Jobs & Webhooks?
-
-- **No Polling Required** - Get notified when crawls complete instead of constantly checking status
-- **Better Resource Usage** - Free up client connections while jobs run in the background
-- **Scalable Architecture** - Ideal for high-volume crawling with TypeScript/Node.js clients or microservices
-- **Reliable Delivery** - Automatic retry with exponential backoff (5 attempts: 1s → 2s → 4s → 8s → 16s)
-
-#### How It Works
-
-1. **Submit Job** → POST to `/crawl/job` with optional `webhook_config`
-2. **Get Task ID** → Receive a `task_id` immediately
-3. **Job Runs** → Crawl executes in the background
-4. **Webhook Fired** → Server POSTs completion notification to your webhook URL
-5. **Fetch Results** → If data wasn't included in webhook, GET `/crawl/job/{task_id}`
-
-#### Quick Example
-
-```bash
-# Submit a crawl job with webhook notification
-curl -X POST http://localhost:11235/crawl/job \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://example.com"],
-    "webhook_config": {
-      "webhook_url": "https://myapp.com/webhooks/crawl-complete",
-      "webhook_data_in_payload": false
-    }
-  }'
-
-# Response: {"task_id": "crawl_a1b2c3d4"}
-```
-
-**Your webhook receives:**
-```json
-{
-  "task_id": "crawl_a1b2c3d4",
-  "task_type": "crawl",
-  "status": "completed",
-  "timestamp": "2025-10-21T10:30:00.000000+00:00",
-  "urls": ["https://example.com"]
-}
-```
-
-Then fetch the results:
-```bash
-curl http://localhost:11235/crawl/job/crawl_a1b2c3d4
-```
-
-#### Include Data in Webhook
-
-Set `webhook_data_in_payload: true` to receive the full crawl results directly in the webhook:
-
-```bash
-curl -X POST http://localhost:11235/crawl/job \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://example.com"],
-    "webhook_config": {
-      "webhook_url": "https://myapp.com/webhooks/crawl-complete",
-      "webhook_data_in_payload": true
-    }
-  }'
-```
-
-**Your webhook receives the complete data:**
-```json
-{
-  "task_id": "crawl_a1b2c3d4",
-  "task_type": "crawl",
-  "status": "completed",
-  "timestamp": "2025-10-21T10:30:00.000000+00:00",
-  "urls": ["https://example.com"],
-  "data": {
-    "markdown": "...",
-    "html": "...",
-    "links": {...},
-    "metadata": {...}
-  }
-}
-```
-
-#### Webhook Authentication
-
-Add custom headers for authentication:
-
-```json
-{
-  "urls": ["https://example.com"],
-  "webhook_config": {
-    "webhook_url": "https://myapp.com/webhooks/crawl",
-    "webhook_data_in_payload": false,
-    "webhook_headers": {
-      "X-Webhook-Secret": "your-secret-token",
-      "X-Service-ID": "crawl4ai-prod"
-    }
-  }
-}
-```
-
-#### Global Default Webhook
-
-Configure a default webhook URL in `config.yml` for all jobs:
-
-```yaml
-webhooks:
-  enabled: true
-  default_url: "https://myapp.com/webhooks/default"
-  data_in_payload: false
-  retry:
-    max_attempts: 5
-    initial_delay_ms: 1000
-    max_delay_ms: 32000
-    timeout_ms: 30000
-```
-
-Now jobs without `webhook_config` automatically use the default webhook.
-
-#### Job Status Polling (Without Webhooks)
-
-If you prefer polling instead of webhooks, just omit `webhook_config`:
-
-```bash
-# Submit job
-curl -X POST http://localhost:11235/crawl/job \
-  -H "Content-Type: application/json" \
-  -d '{"urls": ["https://example.com"]}'
-# Response: {"task_id": "crawl_xyz"}
-
-# Poll for status
-curl http://localhost:11235/crawl/job/crawl_xyz
-```
-
-The response includes `status` field: `"processing"`, `"completed"`, or `"failed"`.
-
-#### LLM Extraction Jobs with Webhooks
-
-The same webhook system works for LLM extraction jobs via `/llm/job`:
-
-```bash
-# Submit LLM extraction job with webhook
-curl -X POST http://localhost:11235/llm/job \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://example.com/article",
-    "q": "Extract the article title, author, and main points",
-    "provider": "openai/gpt-4o-mini",
-    "webhook_config": {
-      "webhook_url": "https://myapp.com/webhooks/llm-complete",
-      "webhook_data_in_payload": true,
-      "webhook_headers": {
-        "X-Webhook-Secret": "your-secret-token"
-      }
-    }
-  }'
-
-# Response: {"task_id": "llm_1234567890"}
-```
-
-**Your webhook receives:**
-```json
-{
-  "task_id": "llm_1234567890",
-  "task_type": "llm_extraction",
-  "status": "completed",
-  "timestamp": "2025-10-22T12:30:00.000000+00:00",
-  "urls": ["https://example.com/article"],
-  "data": {
-    "extracted_content": {
-      "title": "Understanding Web Scraping",
-      "author": "John Doe",
-      "main_points": ["Point 1", "Point 2", "Point 3"]
-    }
-  }
-}
-```
-
-**Key Differences for LLM Jobs:**
-- Task type is `"llm_extraction"` instead of `"crawl"`
-- Extracted data is in `data.extracted_content`
-- Single URL only (not an array)
-- Supports schema-based extraction with `schema` parameter
-
-> 💡 **Pro tip**: See [WEBHOOK_EXAMPLES.md](./WEBHOOK_EXAMPLES.md) for detailed examples including TypeScript client code, Flask webhook handlers, and failure handling.
-
 ---
 
 ## Metrics & Monitoring
@@ -1013,11 +826,10 @@ We're here to help you succeed with Crawl4AI! Here's how to get support:
 
 In this guide, we've covered everything you need to get started with Crawl4AI's Docker deployment:
 - Building and running the Docker container
-- Configuring the environment
+- Configuring the environment  
 - Using the interactive playground for testing
 - Making API requests with proper typing
 - Using the Python SDK
-- Asynchronous job queues with webhook notifications
 - Leveraging specialized endpoints for screenshots, PDFs, and JavaScript execution
 - Connecting via the Model Context Protocol (MCP)
 - Monitoring your deployment

deploy/docker/WEBHOOK_EXAMPLES.md

@@ -1,378 +0,0 @@
-# Webhook Feature Examples
-
-This document provides examples of how to use the webhook feature for crawl jobs in Crawl4AI.
-
-## Overview
-
-The webhook feature allows you to receive notifications when crawl jobs complete, eliminating the need for polling. Webhooks are sent with exponential backoff retry logic to ensure reliable delivery.
-
-## Configuration
-
-### Global Configuration (config.yml)
-
-You can configure default webhook settings in `config.yml`:
-
-```yaml
-webhooks:
-  enabled: true
-  default_url: null  # Optional: default webhook URL for all jobs
-  data_in_payload: false  # Optional: default behavior for including data
-  retry:
-    max_attempts: 5
-    initial_delay_ms: 1000  # 1s, 2s, 4s, 8s, 16s exponential backoff
-    max_delay_ms: 32000
-    timeout_ms: 30000  # 30s timeout per webhook call
-  headers:  # Optional: default headers to include
-    User-Agent: "Crawl4AI-Webhook/1.0"
-```
-
-## API Usage Examples
-
-### Example 1: Basic Webhook (Notification Only)
-
-Send a webhook notification without including the crawl data in the payload.
-
-**Request:**
-```bash
-curl -X POST http://localhost:11235/crawl/job \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://example.com"],
-    "webhook_config": {
-      "webhook_url": "https://myapp.com/webhooks/crawl-complete",
-      "webhook_data_in_payload": false
-    }
-  }'
-```
-
-**Response:**
-```json
-{
-  "task_id": "crawl_a1b2c3d4"
-}
-```
-
-**Webhook Payload Received:**
-```json
-{
-  "task_id": "crawl_a1b2c3d4",
-  "task_type": "crawl",
-  "status": "completed",
-  "timestamp": "2025-10-21T10:30:00.000000+00:00",
-  "urls": ["https://example.com"]
-}
-```
-
-Your webhook handler should then fetch the results:
-```bash
-curl http://localhost:11235/crawl/job/crawl_a1b2c3d4
-```
-
-### Example 2: Webhook with Data Included
-
-Include the full crawl results in the webhook payload.
-
-**Request:**
-```bash
-curl -X POST http://localhost:11235/crawl/job \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://example.com"],
-    "webhook_config": {
-      "webhook_url": "https://myapp.com/webhooks/crawl-complete",
-      "webhook_data_in_payload": true
-    }
-  }'
-```
-
-**Webhook Payload Received:**
-```json
-{
-  "task_id": "crawl_a1b2c3d4",
-  "task_type": "crawl",
-  "status": "completed",
-  "timestamp": "2025-10-21T10:30:00.000000+00:00",
-  "urls": ["https://example.com"],
-  "data": {
-    "markdown": "...",
-    "html": "...",
-    "links": {...},
-    "metadata": {...}
-  }
-}
-```
-
-### Example 3: Webhook with Custom Headers
-
-Include custom headers for authentication or identification.
-
-**Request:**
-```bash
-curl -X POST http://localhost:11235/crawl/job \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://example.com"],
-    "webhook_config": {
-      "webhook_url": "https://myapp.com/webhooks/crawl-complete",
-      "webhook_data_in_payload": false,
-      "webhook_headers": {
-        "X-Webhook-Secret": "my-secret-token",
-        "X-Service-ID": "crawl4ai-production"
-      }
-    }
-  }'
-```
-
-The webhook will be sent with these additional headers plus the default headers from config.
-
-### Example 4: Failure Notification
-
-When a crawl job fails, a webhook is sent with error details.
-
-**Webhook Payload on Failure:**
-```json
-{
-  "task_id": "crawl_a1b2c3d4",
-  "task_type": "crawl",
-  "status": "failed",
-  "timestamp": "2025-10-21T10:30:00.000000+00:00",
-  "urls": ["https://example.com"],
-  "error": "Connection timeout after 30s"
-}
-```
-
-### Example 5: Using Global Default Webhook
-
-If you set a `default_url` in config.yml, jobs without webhook_config will use it:
-
-**config.yml:**
-```yaml
-webhooks:
-  enabled: true
-  default_url: "https://myapp.com/webhooks/default"
-  data_in_payload: false
-```
-
-**Request (no webhook_config needed):**
-```bash
-curl -X POST http://localhost:11235/crawl/job \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://example.com"]
-  }'
-```
-
-The webhook will be sent to the default URL configured in config.yml.
-
-### Example 6: LLM Extraction Job with Webhook
-
-Use webhooks with the LLM extraction endpoint for asynchronous processing.
-
-**Request:**
-```bash
-curl -X POST http://localhost:11235/llm/job \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://example.com/article",
-    "q": "Extract the article title, author, and publication date",
-    "schema": "{\"type\": \"object\", \"properties\": {\"title\": {\"type\": \"string\"}, \"author\": {\"type\": \"string\"}, \"date\": {\"type\": \"string\"}}}",
-    "cache": false,
-    "provider": "openai/gpt-4o-mini",
-    "webhook_config": {
-      "webhook_url": "https://myapp.com/webhooks/llm-complete",
-      "webhook_data_in_payload": true
-    }
-  }'
-```
-
-**Response:**
-```json
-{
-  "task_id": "llm_1698765432_12345"
-}
-```
-
-**Webhook Payload Received:**
-```json
-{
-  "task_id": "llm_1698765432_12345",
-  "task_type": "llm_extraction",
-  "status": "completed",
-  "timestamp": "2025-10-21T10:30:00.000000+00:00",
-  "urls": ["https://example.com/article"],
-  "data": {
-    "extracted_content": {
-      "title": "Understanding Web Scraping",
-      "author": "John Doe",
-      "date": "2025-10-21"
-    }
-  }
-}
-```
-
-## Webhook Handler Example
-
-Here's a simple Python Flask webhook handler that supports both crawl and LLM extraction jobs:
-
-```python
-from flask import Flask, request, jsonify
-import requests
-
-app = Flask(__name__)
-
-@app.route('/webhooks/crawl-complete', methods=['POST'])
-def handle_crawl_webhook():
-    payload = request.json
-
-    task_id = payload['task_id']
-    task_type = payload['task_type']
-    status = payload['status']
-
-    if status == 'completed':
-        # If data not in payload, fetch it
-        if 'data' not in payload:
-            # Determine endpoint based on task type
-            endpoint = 'crawl' if task_type == 'crawl' else 'llm'
-            response = requests.get(f'http://localhost:11235/{endpoint}/job/{task_id}')
-            data = response.json()
-        else:
-            data = payload['data']
-
-        # Process based on task type
-        if task_type == 'crawl':
-            print(f"Processing crawl results for {task_id}")
-            # Handle crawl results
-            results = data.get('results', [])
-            for result in results:
-                print(f"  - {result.get('url')}: {len(result.get('markdown', ''))} chars")
-
-        elif task_type == 'llm_extraction':
-            print(f"Processing LLM extraction for {task_id}")
-            # Handle LLM extraction
-            # Note: Webhook sends 'extracted_content', API returns 'result'
-            extracted = data.get('extracted_content', data.get('result', {}))
-            print(f"  - Extracted: {extracted}")
-
-        # Your business logic here...
-
-    elif status == 'failed':
-        error = payload.get('error', 'Unknown error')
-        print(f"{task_type} job {task_id} failed: {error}")
-        # Handle failure...
-
-    return jsonify({"status": "received"}), 200
-
-if __name__ == '__main__':
-    app.run(port=8080)
-```
-
-## Retry Logic
-
-The webhook delivery service uses exponential backoff retry logic:
-
-- **Attempts:** Up to 5 attempts by default
-- **Delays:** 1s → 2s → 4s → 8s → 16s
-- **Timeout:** 30 seconds per attempt
-- **Retry Conditions:**
-  - Server errors (5xx status codes)
-  - Network errors
-  - Timeouts
-- **No Retry:**
-  - Client errors (4xx status codes)
-  - Successful delivery (2xx status codes)
-
-## Benefits
-
-1. **No Polling Required** - Eliminates constant API calls to check job status
-2. **Real-time Notifications** - Immediate notification when jobs complete
-3. **Reliable Delivery** - Exponential backoff ensures webhooks are delivered
-4. **Flexible** - Choose between notification-only or full data delivery
-5. **Secure** - Support for custom headers for authentication
-6. **Configurable** - Global defaults or per-job configuration
-7. **Universal Support** - Works with both `/crawl/job` and `/llm/job` endpoints
-
-## TypeScript Client Example
-
-```typescript
-interface WebhookConfig {
-  webhook_url: string;
-  webhook_data_in_payload?: boolean;
-  webhook_headers?: Record<string, string>;
-}
-
-interface CrawlJobRequest {
-  urls: string[];
-  browser_config?: Record<string, any>;
-  crawler_config?: Record<string, any>;
-  webhook_config?: WebhookConfig;
-}
-
-interface LLMJobRequest {
-  url: string;
-  q: string;
-  schema?: string;
-  cache?: boolean;
-  provider?: string;
-  webhook_config?: WebhookConfig;
-}
-
-async function createCrawlJob(request: CrawlJobRequest) {
-  const response = await fetch('http://localhost:11235/crawl/job', {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify(request)
-  });
-
-  const { task_id } = await response.json();
-  return task_id;
-}
-
-async function createLLMJob(request: LLMJobRequest) {
-  const response = await fetch('http://localhost:11235/llm/job', {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify(request)
-  });
-
-  const { task_id } = await response.json();
-  return task_id;
-}
-
-// Usage - Crawl Job
-const crawlTaskId = await createCrawlJob({
-  urls: ['https://example.com'],
-  webhook_config: {
-    webhook_url: 'https://myapp.com/webhooks/crawl-complete',
-    webhook_data_in_payload: false,
-    webhook_headers: {
-      'X-Webhook-Secret': 'my-secret'
-    }
-  }
-});
-
-// Usage - LLM Extraction Job
-const llmTaskId = await createLLMJob({
-  url: 'https://example.com/article',
-  q: 'Extract the main points from this article',
-  provider: 'openai/gpt-4o-mini',
-  webhook_config: {
-    webhook_url: 'https://myapp.com/webhooks/llm-complete',
-    webhook_data_in_payload: true,
-    webhook_headers: {
-      'X-Webhook-Secret': 'my-secret'
-    }
-  }
-});
-```
-
-## Monitoring and Debugging
-
-Webhook delivery attempts are logged at INFO level:
-- Successful deliveries
-- Retry attempts with delays
-- Final failures after max attempts
-
-Check the application logs for webhook delivery status:
-```bash
-docker logs crawl4ai-container | grep -i webhook
-```

deploy/docker/__init__.py

@@ -1 +0,0 @@
-# Deploy docker module

deploy/docker/api.py

@@ -46,7 +46,6 @@ from utils import (
     get_llm_temperature,
     get_llm_base_url
 )
-from webhook import WebhookDeliveryService
 
 import psutil, time
 
@@ -128,14 +127,10 @@ async def process_llm_extraction(
     schema: Optional[str] = None,
     cache: str = "0",
     provider: Optional[str] = None,
-    webhook_config: Optional[Dict] = None,
     temperature: Optional[float] = None,
     base_url: Optional[str] = None
 ) -> None:
     """Process LLM extraction in background."""
-    # Initialize webhook service
-    webhook_service = WebhookDeliveryService(config)
-
     try:
         # Validate provider
         is_valid, error_msg = validate_llm_provider(config, provider)
@@ -144,16 +139,6 @@ async def process_llm_extraction(
                 "status": TaskStatus.FAILED,
                 "error": error_msg
             })
-
-            # Send webhook notification on failure
-            await webhook_service.notify_job_completion(
-                task_id=task_id,
-                task_type="llm_extraction",
-                status="failed",
-                urls=[url],
-                webhook_config=webhook_config,
-                error=error_msg
-            )
             return
         api_key = get_llm_api_key(config, provider)  # Returns None to let litellm handle it
         llm_strategy = LLMExtractionStrategy(
@@ -184,40 +169,17 @@ async def process_llm_extraction(
                 "status": TaskStatus.FAILED,
                 "error": result.error_message
             })
-
-            # Send webhook notification on failure
-            await webhook_service.notify_job_completion(
-                task_id=task_id,
-                task_type="llm_extraction",
-                status="failed",
-                urls=[url],
-                webhook_config=webhook_config,
-                error=result.error_message
-            )
             return
 
         try:
             content = json.loads(result.extracted_content)
         except json.JSONDecodeError:
             content = result.extracted_content
-
-        result_data = {"extracted_content": content}
-
         await redis.hset(f"task:{task_id}", mapping={
             "status": TaskStatus.COMPLETED,
             "result": json.dumps(content)
         })
 
-        # Send webhook notification on successful completion
-        await webhook_service.notify_job_completion(
-            task_id=task_id,
-            task_type="llm_extraction",
-            status="completed",
-            urls=[url],
-            webhook_config=webhook_config,
-            result=result_data
-        )
-
     except Exception as e:
         logger.error(f"LLM extraction error: {str(e)}", exc_info=True)
         await redis.hset(f"task:{task_id}", mapping={
@@ -225,16 +187,6 @@ async def process_llm_extraction(
             "error": str(e)
         })
 
-        # Send webhook notification on failure
-        await webhook_service.notify_job_completion(
-            task_id=task_id,
-            task_type="llm_extraction",
-            status="failed",
-            urls=[url],
-            webhook_config=webhook_config,
-            error=str(e)
-        )
-
 async def handle_markdown_request(
     url: str,
     filter_type: FilterType,
@@ -323,7 +275,6 @@ async def handle_llm_request(
     cache: str = "0",
     config: Optional[dict] = None,
     provider: Optional[str] = None,
-    webhook_config: Optional[Dict] = None,
     temperature: Optional[float] = None,
     api_base_url: Optional[str] = None
 ) -> JSONResponse:
@@ -357,7 +308,6 @@ async def handle_llm_request(
             base_url,
             config,
             provider,
-            webhook_config,
             temperature,
             api_base_url
         )
@@ -405,7 +355,6 @@ async def create_new_task(
     base_url: str,
     config: dict,
     provider: Optional[str] = None,
-    webhook_config: Optional[Dict] = None,
     temperature: Optional[float] = None,
     api_base_url: Optional[str] = None
 ) -> JSONResponse:
@@ -416,18 +365,12 @@ async def create_new_task(
 
     from datetime import datetime
     task_id = f"llm_{int(datetime.now().timestamp())}_{id(background_tasks)}"
-
+    
-    task_data = {
+    await redis.hset(f"task:{task_id}", mapping={
         "status": TaskStatus.PROCESSING,
         "created_at": datetime.now().isoformat(),
         "url": decoded_url
-    }
+    })
-
-    # Store webhook config if provided
-    if webhook_config:
-        task_data["webhook_config"] = json.dumps(webhook_config)
-
-    await redis.hset(f"task:{task_id}", mapping=task_data)
 
     background_tasks.add_task(
         process_llm_extraction,
@@ -439,7 +382,6 @@ async def create_new_task(
         schema,
         cache,
         provider,
-        webhook_config,
         temperature,
         api_base_url
     )
@@ -781,7 +723,6 @@ async def handle_crawl_job(
     browser_config: Dict,
     crawler_config: Dict,
     config: Dict,
-    webhook_config: Optional[Dict] = None,
 ) -> Dict:
     """
     Fire-and-forget version of handle_crawl_request.
@@ -789,24 +730,13 @@ async def handle_crawl_job(
     lets /crawl/job/{task_id} polling fetch the result.
     """
     task_id = f"crawl_{uuid4().hex[:8]}"
-
+    await redis.hset(f"task:{task_id}", mapping={
-    # Store task data in Redis
-    task_data = {
         "status": TaskStatus.PROCESSING,         # <-- keep enum values consistent
         "created_at": datetime.now(timezone.utc).replace(tzinfo=None).isoformat(),
         "url": json.dumps(urls),                 # store list as JSON string
         "result": "",
         "error": "",
-    }
+    })
-
-    # Store webhook config if provided
-    if webhook_config:
-        task_data["webhook_config"] = json.dumps(webhook_config)
-
-    await redis.hset(f"task:{task_id}", mapping=task_data)
-
-    # Initialize webhook service
-    webhook_service = WebhookDeliveryService(config)
 
     async def _runner():
         try:
@@ -820,17 +750,6 @@ async def handle_crawl_job(
                 "status": TaskStatus.COMPLETED,
                 "result": json.dumps(result),
             })
-
-            # Send webhook notification on successful completion
-            await webhook_service.notify_job_completion(
-                task_id=task_id,
-                task_type="crawl",
-                status="completed",
-                urls=urls,
-                webhook_config=webhook_config,
-                result=result
-            )
-
             await asyncio.sleep(5)  # Give Redis time to process the update
         except Exception as exc:
             await redis.hset(f"task:{task_id}", mapping={
@@ -838,15 +757,5 @@ async def handle_crawl_job(
                 "error": str(exc),
             })
 
-            # Send webhook notification on failure
-            await webhook_service.notify_job_completion(
-                task_id=task_id,
-                task_type="crawl",
-                status="failed",
-                urls=urls,
-                webhook_config=webhook_config,
-                error=str(exc)
-            )
-
     background_tasks.add_task(_runner)
     return {"task_id": task_id}

deploy/docker/cnode_cli.py

@@ -1,492 +0,0 @@
-"""
-Crawl4AI Server CLI Commands
-
-Provides `cnode` command group for Docker orchestration.
-"""
-
-import click
-import anyio
-from rich.console import Console
-from rich.table import Table
-from rich.panel import Panel
-from rich.prompt import Confirm
-
-from deploy.docker.server_manager import ServerManager
-
-
-console = Console()
-
-
-@click.group()
-def cli():
-    """Manage Crawl4AI Docker server instances
-
-    \b
-    One-command deployment with automatic scaling:
-      • Single container for development (N=1)
-      • Docker Swarm for production with built-in load balancing (N>1)
-      • Docker Compose + Nginx as fallback (N>1)
-
-    \b
-    Examples:
-      cnode start                    # Single container on port 11235
-      cnode start --replicas 3       # Auto-detect Swarm or Compose
-      cnode start -r 5 --port 8080   # 5 replicas on custom port
-      cnode status                   # Check current deployment
-      cnode scale 10                 # Scale to 10 replicas
-      cnode stop                     # Stop and cleanup
-    """
-    pass
-
-
-@cli.command("start")
-@click.option(
-    "--replicas", "-r",
-    type=int,
-    default=1,
-    help="Number of container replicas (default: 1)"
-)
-@click.option(
-    "--mode",
-    type=click.Choice(["auto", "single", "swarm", "compose"]),
-    default="auto",
-    help="Deployment mode (default: auto-detect)"
-)
-@click.option(
-    "--port", "-p",
-    type=int,
-    default=11235,
-    help="External port to expose (default: 11235)"
-)
-@click.option(
-    "--env-file",
-    type=click.Path(exists=True),
-    help="Path to environment file"
-)
-@click.option(
-    "--image",
-    default="unclecode/crawl4ai:latest",
-    help="Docker image to use (default: unclecode/crawl4ai:latest)"
-)
-def start_cmd(replicas: int, mode: str, port: int, env_file: str, image: str):
-    """Start Crawl4AI server with automatic orchestration.
-
-    Deployment modes:
-    - auto: Automatically choose best mode (default)
-    - single: Single container (N=1 only)
-    - swarm: Docker Swarm with built-in load balancing
-    - compose: Docker Compose + Nginx reverse proxy
-
-    The server will:
-    1. Check if Docker is running
-    2. Validate port availability
-    3. Pull image if needed
-    4. Start container(s) with health checks
-    5. Save state for management
-
-    Examples:
-        # Development: single container
-        cnode start
-
-        # Production: 5 replicas with Swarm
-        cnode start --replicas 5
-
-        # Custom configuration
-        cnode start -r 3 --port 8080 --env-file .env.prod
-    """
-    manager = ServerManager()
-
-    console.print(Panel(
-        f"[cyan]Starting Crawl4AI Server[/cyan]\n\n"
-        f"Replicas: [yellow]{replicas}[/yellow]\n"
-        f"Mode: [yellow]{mode}[/yellow]\n"
-        f"Port: [yellow]{port}[/yellow]\n"
-        f"Image: [yellow]{image}[/yellow]",
-        title="Server Start",
-        border_style="cyan"
-    ))
-
-    with console.status("[cyan]Starting server..."):
-        async def _start():
-            return await manager.start(
-                replicas=replicas,
-                mode=mode,
-                port=port,
-                env_file=env_file,
-                image=image
-            )
-        result = anyio.run(_start)
-
-    if result["success"]:
-        console.print(Panel(
-            f"[green]✓ Server started successfully![/green]\n\n"
-            f"Mode: [cyan]{result.get('state_data', {}).get('mode', mode)}[/cyan]\n"
-            f"URL: [bold]http://localhost:{port}[/bold]\n"
-            f"Health: [bold]http://localhost:{port}/health[/bold]\n"
-            f"Monitor: [bold]http://localhost:{port}/monitor[/bold]",
-            title="Server Running",
-            border_style="green"
-        ))
-    else:
-        error_msg = result.get("error", result.get("message", "Unknown error"))
-        console.print(Panel(
-            f"[red]✗ Failed to start server[/red]\n\n"
-            f"{error_msg}",
-            title="Error",
-            border_style="red"
-        ))
-
-        if "already running" in error_msg.lower():
-            console.print("\n[yellow]Hint: Use 'cnode status' to check current deployment[/yellow]")
-            console.print("[yellow]      Use 'cnode stop' to stop existing server[/yellow]")
-
-
-@cli.command("status")
-def status_cmd():
-    """Show current server status and deployment info.
-
-    Displays:
-    - Running state (up/down)
-    - Deployment mode (single/swarm/compose)
-    - Number of replicas
-    - Port mapping
-    - Uptime
-    - Image version
-
-    Example:
-        cnode status
-    """
-    manager = ServerManager()
-
-    async def _status():
-        return await manager.status()
-    result = anyio.run(_status)
-
-    if result["running"]:
-        table = Table(title="Crawl4AI Server Status", border_style="green")
-        table.add_column("Property", style="cyan")
-        table.add_column("Value", style="green")
-
-        table.add_row("Status", "🟢 Running")
-        table.add_row("Mode", result["mode"])
-        table.add_row("Replicas", str(result.get("replicas", 1)))
-        table.add_row("Port", str(result.get("port", 11235)))
-        table.add_row("Image", result.get("image", "unknown"))
-        table.add_row("Uptime", result.get("uptime", "unknown"))
-        table.add_row("Started", result.get("started_at", "unknown"))
-
-        console.print(table)
-        console.print(f"\n[green]✓ Server is healthy[/green]")
-        console.print(f"[dim]Access: http://localhost:{result.get('port', 11235)}[/dim]")
-    else:
-        console.print(Panel(
-            f"[yellow]No server is currently running[/yellow]\n\n"
-            f"Use 'cnode start' to launch a server",
-            title="Server Status",
-            border_style="yellow"
-        ))
-
-
-@cli.command("stop")
-@click.option(
-    "--remove-volumes",
-    is_flag=True,
-    help="Remove associated volumes (WARNING: deletes data)"
-)
-def stop_cmd(remove_volumes: bool):
-    """Stop running Crawl4AI server and cleanup resources.
-
-    This will:
-    1. Stop all running containers/services
-    2. Remove containers
-    3. Optionally remove volumes (--remove-volumes)
-    4. Clean up state files
-
-    WARNING: Use --remove-volumes with caution as it will delete
-    persistent data including Redis databases and logs.
-
-    Examples:
-        # Stop server, keep volumes
-        cnode stop
-
-        # Stop and remove all data
-        cnode stop --remove-volumes
-    """
-    manager = ServerManager()
-
-    # Confirm if removing volumes
-    if remove_volumes:
-        if not Confirm.ask(
-            "[red]⚠️  This will delete all server data including Redis databases. Continue?[/red]"
-        ):
-            console.print("[yellow]Cancelled[/yellow]")
-            return
-
-    with console.status("[cyan]Stopping server..."):
-        async def _stop():
-            return await manager.stop(remove_volumes=remove_volumes)
-        result = anyio.run(_stop)
-
-    if result["success"]:
-        console.print(Panel(
-            f"[green]✓ Server stopped successfully[/green]\n\n"
-            f"{result.get('message', 'All resources cleaned up')}",
-            title="Server Stopped",
-            border_style="green"
-        ))
-    else:
-        console.print(Panel(
-            f"[red]✗ Error stopping server[/red]\n\n"
-            f"{result.get('error', result.get('message', 'Unknown error'))}",
-            title="Error",
-            border_style="red"
-        ))
-
-
-@cli.command("scale")
-@click.argument("replicas", type=int)
-def scale_cmd(replicas: int):
-    """Scale server to specified number of replicas.
-
-    Only works with Swarm or Compose modes. Single container
-    mode cannot be scaled (must stop and restart with --replicas).
-
-    Scaling is live and does not require downtime. The load
-    balancer will automatically distribute traffic to new replicas.
-
-    Examples:
-        # Scale up to 10 replicas
-        cnode scale 10
-
-        # Scale down to 2 replicas
-        cnode scale 2
-
-        # Scale to 1 (minimum)
-        cnode scale 1
-    """
-    if replicas < 1:
-        console.print("[red]Error: Replicas must be at least 1[/red]")
-        return
-
-    manager = ServerManager()
-
-    with console.status(f"[cyan]Scaling to {replicas} replicas..."):
-        async def _scale():
-            return await manager.scale(replicas=replicas)
-        result = anyio.run(_scale)
-
-    if result["success"]:
-        console.print(Panel(
-            f"[green]✓ Scaled successfully[/green]\n\n"
-            f"New replica count: [bold]{replicas}[/bold]\n"
-            f"Mode: [cyan]{result.get('mode')}[/cyan]",
-            title="Scaling Complete",
-            border_style="green"
-        ))
-    else:
-        error_msg = result.get("error", result.get("message", "Unknown error"))
-        console.print(Panel(
-            f"[red]✗ Scaling failed[/red]\n\n"
-            f"{error_msg}",
-            title="Error",
-            border_style="red"
-        ))
-
-        if "single container" in error_msg.lower():
-            console.print("\n[yellow]Hint: For single container mode:[/yellow]")
-            console.print("[yellow]  1. cnode stop[/yellow]")
-            console.print(f"[yellow]  2. cnode start --replicas {replicas}[/yellow]")
-
-
-@cli.command("logs")
-@click.option(
-    "--follow", "-f",
-    is_flag=True,
-    help="Follow log output (like tail -f)"
-)
-@click.option(
-    "--tail",
-    type=int,
-    default=100,
-    help="Number of lines to show (default: 100)"
-)
-def logs_cmd(follow: bool, tail: int):
-    """View server logs.
-
-    Shows logs from running containers/services. Use --follow
-    to stream logs in real-time.
-
-    Examples:
-        # Show last 100 lines
-        cnode logs
-
-        # Show last 500 lines
-        cnode logs --tail 500
-
-        # Follow logs in real-time
-        cnode logs --follow
-
-        # Combine options
-        cnode logs -f --tail 50
-    """
-    manager = ServerManager()
-
-    async def _logs():
-        return await manager.logs(follow=follow, tail=tail)
-    output = anyio.run(_logs)
-    console.print(output)
-
-
-@cli.command("cleanup")
-@click.option(
-    "--force",
-    is_flag=True,
-    help="Force cleanup even if state file doesn't exist"
-)
-def cleanup_cmd(force: bool):
-    """Force cleanup of all Crawl4AI Docker resources.
-
-    Stops and removes all containers, networks, and optionally volumes.
-    Useful when server is stuck or state is corrupted.
-
-    Examples:
-        # Clean up everything
-        cnode cleanup
-
-        # Force cleanup (ignore state file)
-        cnode cleanup --force
-    """
-    manager = ServerManager()
-
-    console.print(Panel(
-        f"[yellow]⚠️  Cleaning up Crawl4AI Docker resources[/yellow]\n\n"
-        f"This will stop and remove:\n"
-        f"- All Crawl4AI containers\n"
-        f"- Nginx load balancer\n"
-        f"- Redis instance\n"
-        f"- Docker networks\n"
-        f"- State files",
-        title="Cleanup",
-        border_style="yellow"
-    ))
-
-    if not force and not Confirm.ask("[yellow]Continue with cleanup?[/yellow]"):
-        console.print("[yellow]Cancelled[/yellow]")
-        return
-
-    with console.status("[cyan]Cleaning up resources..."):
-        async def _cleanup():
-            return await manager.cleanup(force=force)
-        result = anyio.run(_cleanup)
-
-    if result["success"]:
-        console.print(Panel(
-            f"[green]✓ Cleanup completed successfully[/green]\n\n"
-            f"Removed: {result.get('removed', 0)} containers\n"
-            f"{result.get('message', 'All resources cleaned up')}",
-            title="Cleanup Complete",
-            border_style="green"
-        ))
-    else:
-        console.print(Panel(
-            f"[yellow]⚠️  Partial cleanup[/yellow]\n\n"
-            f"{result.get('message', 'Some resources may still exist')}",
-            title="Cleanup Status",
-            border_style="yellow"
-        ))
-
-
-@cli.command("restart")
-@click.option(
-    "--replicas", "-r",
-    type=int,
-    help="New replica count (optional)"
-)
-def restart_cmd(replicas: int):
-    """Restart server (stop then start with same config).
-
-    Preserves existing configuration unless overridden with options.
-    Useful for applying image updates or recovering from errors.
-
-    Examples:
-        # Restart with same configuration
-        cnode restart
-
-        # Restart and change replica count
-        cnode restart --replicas 5
-    """
-    manager = ServerManager()
-
-    # Get current state
-    async def _get_status():
-        return await manager.status()
-    current = anyio.run(_get_status)
-
-    if not current["running"]:
-        console.print("[yellow]No server is running. Use 'cnode start' instead.[/yellow]")
-        return
-
-    # Extract current config
-    current_replicas = current.get("replicas", 1)
-    current_port = current.get("port", 11235)
-    current_image = current.get("image", "unclecode/crawl4ai:latest")
-    current_mode = current.get("mode", "auto")
-
-    # Override with CLI args
-    new_replicas = replicas if replicas is not None else current_replicas
-
-    console.print(Panel(
-        f"[cyan]Restarting Crawl4AI Server[/cyan]\n\n"
-        f"Replicas: [yellow]{current_replicas}[/yellow] → [green]{new_replicas}[/green]\n"
-        f"Port: [yellow]{current_port}[/yellow]\n"
-        f"Mode: [yellow]{current_mode}[/yellow]",
-        title="Server Restart",
-        border_style="cyan"
-    ))
-
-    # Stop current
-    with console.status("[cyan]Stopping current server..."):
-        async def _stop_server():
-            return await manager.stop(remove_volumes=False)
-        stop_result = anyio.run(_stop_server)
-
-    if not stop_result["success"]:
-        console.print(f"[red]Failed to stop server: {stop_result.get('error')}[/red]")
-        return
-
-    # Start new
-    with console.status("[cyan]Starting server..."):
-        async def _start_server():
-            return await manager.start(
-                replicas=new_replicas,
-                mode="auto",
-                port=current_port,
-                image=current_image
-            )
-        start_result = anyio.run(_start_server)
-
-    if start_result["success"]:
-        console.print(Panel(
-            f"[green]✓ Server restarted successfully![/green]\n\n"
-            f"URL: [bold]http://localhost:{current_port}[/bold]",
-            title="Restart Complete",
-            border_style="green"
-        ))
-    else:
-        console.print(Panel(
-            f"[red]✗ Failed to restart server[/red]\n\n"
-            f"{start_result.get('error', 'Unknown error')}",
-            title="Error",
-            border_style="red"
-        ))
-
-
-def main():
-    """Entry point for cnode CLI"""
-    cli()
-
-
-if __name__ == "__main__":
-    main()
-
-# Test comment

deploy/docker/config.yml

@@ -87,17 +87,4 @@ observability:
     enabled: True
     endpoint: "/metrics"
   health_check:
-    endpoint: "/health"
+    endpoint: "/health"
-
-# Webhook Configuration
-webhooks:
-  enabled: true
-  default_url: null  # Optional: default webhook URL for all jobs
-  data_in_payload: false  # Optional: default behavior for including data
-  retry:
-    max_attempts: 5
-    initial_delay_ms: 1000  # 1s, 2s, 4s, 8s, 16s exponential backoff
-    max_delay_ms: 32000
-    timeout_ms: 30000  # 30s timeout per webhook call
-  headers:  # Optional: default headers to include
-    User-Agent: "Crawl4AI-Webhook/1.0"

deploy/docker/job.py

@@ -12,7 +12,6 @@ from api import (
     handle_crawl_job,
     handle_task_status,
 )
-from schemas import WebhookConfig
 
 # ------------- dependency placeholders -------------
 _redis = None        # will be injected from server.py
@@ -38,7 +37,6 @@ class LlmJobPayload(BaseModel):
     schema: Optional[str] = None
     cache:  bool = False
     provider: Optional[str] = None
-    webhook_config: Optional[WebhookConfig] = None
     temperature: Optional[float] = None
     base_url: Optional[str] = None
 
@@ -47,7 +45,6 @@ class CrawlJobPayload(BaseModel):
     urls:           list[HttpUrl]
     browser_config: Dict = {}
     crawler_config: Dict = {}
-    webhook_config: Optional[WebhookConfig] = None
 
 
 # ---------- LL​M job ---------------------------------------------------------
@@ -58,10 +55,6 @@ async def llm_job_enqueue(
         request: Request,
         _td: Dict = Depends(lambda: _token_dep()),   # late-bound dep
 ):
-    webhook_config = None
-    if payload.webhook_config:
-        webhook_config = payload.webhook_config.model_dump(mode='json')
-
     return await handle_llm_request(
         _redis,
         background_tasks,
@@ -72,7 +65,6 @@ async def llm_job_enqueue(
         cache=payload.cache,
         config=_config,
         provider=payload.provider,
-        webhook_config=webhook_config,
         temperature=payload.temperature,
         api_base_url=payload.base_url,
     )
@@ -94,10 +86,6 @@ async def crawl_job_enqueue(
         background_tasks: BackgroundTasks,
         _td: Dict = Depends(lambda: _token_dep()),
 ):
-    webhook_config = None
-    if payload.webhook_config:
-        webhook_config = payload.webhook_config.model_dump(mode='json')
-
     return await handle_crawl_job(
         _redis,
         background_tasks,
@@ -105,7 +93,6 @@ async def crawl_job_enqueue(
         payload.browser_config,
         payload.crawler_config,
         config=_config,
-        webhook_config=webhook_config,
     )
 
 

deploy/docker/monitor.py

@@ -5,7 +5,6 @@ import asyncio
 from typing import Dict, List, Optional
 from datetime import datetime, timezone
 from collections import deque
-from dataclasses import dataclass
 from redis import asyncio as aioredis
 from utils import get_container_memory_percent
 import psutil
@@ -13,49 +12,13 @@ import logging
 
 logger = logging.getLogger(__name__)
 
-
-# ========== Configuration ==========
-
-@dataclass
-class RedisTTLConfig:
-    """Redis TTL configuration (in seconds).
-
-    Configures how long different types of monitoring data are retained in Redis.
-    Adjust based on your monitoring needs and Redis memory constraints.
-    """
-    active_requests: int = 300  # 5 minutes - short-lived active request data
-    completed_requests: int = 3600  # 1 hour - recent completed requests
-    janitor_events: int = 3600  # 1 hour - browser cleanup events
-    errors: int = 3600  # 1 hour - error logs
-    endpoint_stats: int = 86400  # 24 hours - aggregated endpoint statistics
-    heartbeat: int = 60  # 1 minute - container heartbeat (2x the 30s interval)
-
-    @classmethod
-    def from_env(cls) -> 'RedisTTLConfig':
-        """Load TTL configuration from environment variables."""
-        import os
-        return cls(
-            active_requests=int(os.getenv('REDIS_TTL_ACTIVE_REQUESTS', 300)),
-            completed_requests=int(os.getenv('REDIS_TTL_COMPLETED_REQUESTS', 3600)),
-            janitor_events=int(os.getenv('REDIS_TTL_JANITOR_EVENTS', 3600)),
-            errors=int(os.getenv('REDIS_TTL_ERRORS', 3600)),
-            endpoint_stats=int(os.getenv('REDIS_TTL_ENDPOINT_STATS', 86400)),
-            heartbeat=int(os.getenv('REDIS_TTL_HEARTBEAT', 60)),
-        )
-
-
 class MonitorStats:
     """Tracks real-time server stats with Redis persistence."""
 
-    def __init__(self, redis: aioredis.Redis, ttl_config: Optional[RedisTTLConfig] = None):
+    def __init__(self, redis: aioredis.Redis):
         self.redis = redis
-        self.ttl = ttl_config or RedisTTLConfig.from_env()
         self.start_time = time.time()
 
-        # Get container ID for Redis keys
-        from utils import get_container_id
-        self.container_id = get_container_id()
-
         # In-memory queues (fast reads, Redis backup)
         self.active_requests: Dict[str, Dict] = {}  # id -> request info
         self.completed_requests: deque = deque(maxlen=100)  # Last 100
@@ -69,9 +32,6 @@ class MonitorStats:
         self._persist_queue: asyncio.Queue = asyncio.Queue(maxsize=10)
         self._persist_worker_task: Optional[asyncio.Task] = None
 
-        # Heartbeat task for container discovery
-        self._heartbeat_task: Optional[asyncio.Task] = None
-
         # Timeline data (5min window, 5s resolution = 60 points)
         self.memory_timeline: deque = deque(maxlen=60)
         self.requests_timeline: deque = deque(maxlen=60)
@@ -85,14 +45,10 @@ class MonitorStats:
             "url": url[:100],  # Truncate long URLs
             "start_time": time.time(),
             "config_sig": config.get("sig", "default") if config else "default",
-            "mem_start": psutil.Process().memory_info().rss / (1024 * 1024),
+            "mem_start": psutil.Process().memory_info().rss / (1024 * 1024)
-            "container_id": self.container_id
         }
         self.active_requests[request_id] = req_info
 
-        # Persist to Redis
-        await self._persist_active_requests()
-
         # Increment endpoint counter
         if endpoint not in self.endpoint_stats:
             self.endpoint_stats[endpoint] = {
@@ -139,29 +95,19 @@ class MonitorStats:
             "success": success,
             "error": error,
             "status_code": status_code,
-            "pool_hit": pool_hit,
+            "pool_hit": pool_hit
-            "container_id": self.container_id
         }
         self.completed_requests.append(completed)
 
-        # Persist to Redis
-        await self._persist_completed_requests()
-        await self._persist_active_requests()  # Update active (removed this request)
-
         # Track errors
         if not success and error:
-            error_entry = {
+            self.errors.append({
                 "timestamp": end_time,
                 "endpoint": endpoint,
                 "url": req_info["url"],
                 "error": error,
-                "request_id": request_id,
+                "request_id": request_id
-                "message": error,
+            })
-                "level": "ERROR",
-                "container_id": self.container_id
-            }
-            self.errors.append(error_entry)
-            await self._persist_errors()
 
         await self._persist_endpoint_stats()
 
@@ -171,10 +117,8 @@ class MonitorStats:
             "timestamp": time.time(),
             "type": event_type,  # "close_cold", "close_hot", "promote"
             "sig": sig[:8],
-            "details": details,
+            "details": details
-            "container_id": self.container_id
         })
-        await self._persist_janitor_events()
 
     def _cleanup_old_entries(self, max_age_seconds: int = 300):
         """Remove entries older than max_age_seconds (default 5min)."""
@@ -205,23 +149,13 @@ class MonitorStats:
         recent_reqs = sum(1 for req in self.completed_requests
                          if now - req.get("end_time", 0) < 5)
 
-        # Browser counts (acquire lock with timeout to prevent deadlock)
+        # Browser counts (acquire lock to prevent race conditions)
         from crawler_pool import PERMANENT, HOT_POOL, COLD_POOL, LOCK
-        try:
+        async with LOCK:
-            async with asyncio.timeout(2.0):
-                async with LOCK:
-                    browser_count = {
-                        "permanent": 1 if PERMANENT else 0,
-                        "hot": len(HOT_POOL),
-                        "cold": len(COLD_POOL)
-                    }
-        except asyncio.TimeoutError:
-            logger.warning("Lock acquisition timeout in update_timeline, using cached browser counts")
-            # Use last known values or defaults
             browser_count = {
-                "permanent": 1,
+                "permanent": 1 if PERMANENT else 0,
-                "hot": 0,
+                "hot": len(HOT_POOL),
-                "cold": 0
+                "cold": len(COLD_POOL)
             }
 
         self.memory_timeline.append({"time": now, "value": mem_pct})
@@ -229,117 +163,15 @@ class MonitorStats:
         self.browser_timeline.append({"time": now, "browsers": browser_count})
 
     async def _persist_endpoint_stats(self):
-        """Persist endpoint stats to Redis with retry logic."""
+        """Persist endpoint stats to Redis."""
-        max_retries = 3
+        try:
-        for attempt in range(max_retries):
+            await self.redis.set(
-            try:
+                "monitor:endpoint_stats",
-                await self.redis.set(
+                json.dumps(self.endpoint_stats),
-                    "monitor:endpoint_stats",
+                ex=86400  # 24h TTL
-                    json.dumps(self.endpoint_stats),
+            )
-                    ex=self.ttl.endpoint_stats
+        except Exception as e:
-                )
+            logger.warning(f"Failed to persist endpoint stats: {e}")
-                return  # Success
-            except aioredis.ConnectionError as e:
-                if attempt < max_retries - 1:
-                    backoff = 0.5 * (2 ** attempt)  # 0.5s, 1s, 2s
-                    logger.warning(f"Redis connection error persisting endpoint stats (attempt {attempt + 1}/{max_retries}), retrying in {backoff}s: {e}")
-                    await asyncio.sleep(backoff)
-                else:
-                    logger.error(f"Failed to persist endpoint stats after {max_retries} attempts: {e}")
-            except Exception as e:
-                logger.error(f"Non-retryable error persisting endpoint stats: {e}")
-                break
-
-    async def _persist_active_requests(self):
-        """Persist active requests to Redis with retry logic."""
-        max_retries = 3
-        for attempt in range(max_retries):
-            try:
-                if self.active_requests:
-                    await self.redis.set(
-                        f"monitor:{self.container_id}:active_requests",
-                        json.dumps(list(self.active_requests.values())),
-                        ex=self.ttl.active_requests
-                    )
-                else:
-                    await self.redis.delete(f"monitor:{self.container_id}:active_requests")
-                return  # Success
-            except aioredis.ConnectionError as e:
-                if attempt < max_retries - 1:
-                    backoff = 0.5 * (2 ** attempt)  # 0.5s, 1s, 2s
-                    logger.warning(f"Redis connection error persisting active requests (attempt {attempt + 1}/{max_retries}), retrying in {backoff}s: {e}")
-                    await asyncio.sleep(backoff)
-                else:
-                    logger.error(f"Failed to persist active requests after {max_retries} attempts: {e}")
-            except Exception as e:
-                logger.error(f"Non-retryable error persisting active requests: {e}")
-                break
-
-    async def _persist_completed_requests(self):
-        """Persist completed requests to Redis with retry logic."""
-        max_retries = 3
-        for attempt in range(max_retries):
-            try:
-                await self.redis.set(
-                    f"monitor:{self.container_id}:completed",
-                    json.dumps(list(self.completed_requests)),
-                    ex=self.ttl.completed_requests
-                )
-                return  # Success
-            except aioredis.ConnectionError as e:
-                if attempt < max_retries - 1:
-                    backoff = 0.5 * (2 ** attempt)  # 0.5s, 1s, 2s
-                    logger.warning(f"Redis connection error persisting completed requests (attempt {attempt + 1}/{max_retries}), retrying in {backoff}s: {e}")
-                    await asyncio.sleep(backoff)
-                else:
-                    logger.error(f"Failed to persist completed requests after {max_retries} attempts: {e}")
-            except Exception as e:
-                logger.error(f"Non-retryable error persisting completed requests: {e}")
-                break
-
-    async def _persist_janitor_events(self):
-        """Persist janitor events to Redis with retry logic."""
-        max_retries = 3
-        for attempt in range(max_retries):
-            try:
-                await self.redis.set(
-                    f"monitor:{self.container_id}:janitor",
-                    json.dumps(list(self.janitor_events)),
-                    ex=self.ttl.janitor_events
-                )
-                return  # Success
-            except aioredis.ConnectionError as e:
-                if attempt < max_retries - 1:
-                    backoff = 0.5 * (2 ** attempt)  # 0.5s, 1s, 2s
-                    logger.warning(f"Redis connection error persisting janitor events (attempt {attempt + 1}/{max_retries}), retrying in {backoff}s: {e}")
-                    await asyncio.sleep(backoff)
-                else:
-                    logger.error(f"Failed to persist janitor events after {max_retries} attempts: {e}")
-            except Exception as e:
-                logger.error(f"Non-retryable error persisting janitor events: {e}")
-                break
-
-    async def _persist_errors(self):
-        """Persist errors to Redis with retry logic."""
-        max_retries = 3
-        for attempt in range(max_retries):
-            try:
-                await self.redis.set(
-                    f"monitor:{self.container_id}:errors",
-                    json.dumps(list(self.errors)),
-                    ex=self.ttl.errors
-                )
-                return  # Success
-            except aioredis.ConnectionError as e:
-                if attempt < max_retries - 1:
-                    backoff = 0.5 * (2 ** attempt)  # 0.5s, 1s, 2s
-                    logger.warning(f"Redis connection error persisting errors (attempt {attempt + 1}/{max_retries}), retrying in {backoff}s: {e}")
-                    await asyncio.sleep(backoff)
-                else:
-                    logger.error(f"Failed to persist errors after {max_retries} attempts: {e}")
-            except Exception as e:
-                logger.error(f"Non-retryable error persisting errors: {e}")
-                break
 
     async def _persistence_worker(self):
         """Background worker to persist stats to Redis."""
@@ -370,121 +202,25 @@ class MonitorStats:
             self._persist_worker_task = None
             logger.info("Stopped persistence worker")
 
-    async def _heartbeat_worker(self):
-        """Send heartbeat to Redis every 30s with circuit breaker for failures."""
-        from utils import detect_deployment_mode
-        import os
-
-        heartbeat_failures = 0
-        max_failures = 5  # Circuit breaker threshold
-
-        while True:
-            try:
-                # Get hostname/container name for friendly display
-                # Try HOSTNAME env var first (set by Docker Compose), then socket.gethostname()
-                import socket
-                hostname = os.getenv("HOSTNAME", socket.gethostname())
-
-                # Register this container
-                mode, containers = detect_deployment_mode()
-                container_info = {
-                    "id": self.container_id,
-                    "hostname": hostname,
-                    "last_seen": time.time(),
-                    "mode": mode,
-                    "failure_count": heartbeat_failures
-                }
-
-                # Set heartbeat with configured TTL
-                await self.redis.setex(
-                    f"monitor:heartbeat:{self.container_id}",
-                    self.ttl.heartbeat,
-                    json.dumps(container_info)
-                )
-
-                # Add to active containers set
-                await self.redis.sadd("monitor:active_containers", self.container_id)
-
-                # Reset failure counter on success
-                heartbeat_failures = 0
-
-                # Wait 30s before next heartbeat
-                await asyncio.sleep(30)
-
-            except asyncio.CancelledError:
-                break
-            except aioredis.ConnectionError as e:
-                heartbeat_failures += 1
-                logger.error(
-                    f"Heartbeat Redis connection error (attempt {heartbeat_failures}/{max_failures}): {e}"
-                )
-
-                if heartbeat_failures >= max_failures:
-                    # Circuit breaker - back off for longer
-                    logger.critical(
-                        f"Heartbeat circuit breaker triggered after {heartbeat_failures} failures. "
-                        f"Container will appear offline for 5 minutes."
-                    )
-                    await asyncio.sleep(300)  # 5 min backoff
-                    heartbeat_failures = 0
-                else:
-                    # Exponential backoff
-                    backoff = min(30 * (2 ** heartbeat_failures), 300)
-                    await asyncio.sleep(backoff)
-            except Exception as e:
-                logger.error(f"Unexpected heartbeat error: {e}", exc_info=True)
-                await asyncio.sleep(30)
-
-    def start_heartbeat(self):
-        """Start the heartbeat worker."""
-        if not self._heartbeat_task:
-            self._heartbeat_task = asyncio.create_task(self._heartbeat_worker())
-            logger.info("Started heartbeat worker")
-
-    async def stop_heartbeat(self):
-        """Stop the heartbeat worker and immediately deregister container."""
-        if self._heartbeat_task:
-            self._heartbeat_task.cancel()
-            try:
-                await self._heartbeat_task
-            except asyncio.CancelledError:
-                pass
-
-            # Immediate deregistration (no 60s wait)
-            try:
-                await self.redis.srem("monitor:active_containers", self.container_id)
-                await self.redis.delete(f"monitor:heartbeat:{self.container_id}")
-                logger.info(f"Container {self.container_id} immediately deregistered from monitoring")
-            except Exception as e:
-                logger.warning(f"Failed to deregister container on shutdown: {e}")
-
-            self._heartbeat_task = None
-            logger.info("Stopped heartbeat worker")
-
     async def cleanup(self):
         """Cleanup on shutdown - persist final stats and stop workers."""
         logger.info("Monitor cleanup starting...")
         try:
             # Persist final stats before shutdown
             await self._persist_endpoint_stats()
-            # Stop background workers
+            # Stop background worker
             await self.stop_persistence_worker()
-            await self.stop_heartbeat()
             logger.info("Monitor cleanup completed")
         except Exception as e:
             logger.error(f"Monitor cleanup error: {e}")
 
     async def load_from_redis(self):
-        """Load persisted stats from Redis and start workers."""
+        """Load persisted stats from Redis."""
         try:
             data = await self.redis.get("monitor:endpoint_stats")
             if data:
                 self.endpoint_stats = json.loads(data)
                 logger.info("Loaded endpoint stats from Redis")
-
-            # Start background workers
-            self.start_heartbeat()
-
         except Exception as e:
             logger.warning(f"Failed to load from Redis: {e}")
 
@@ -496,28 +232,17 @@ class MonitorStats:
         # Network I/O (delta since last call)
         net = psutil.net_io_counters()
 
-        # Pool status (acquire lock with timeout to prevent race conditions)
+        # Pool status (acquire lock to prevent race conditions)
         from crawler_pool import PERMANENT, HOT_POOL, COLD_POOL, LOCK
-        try:
+        async with LOCK:
-            async with asyncio.timeout(2.0):
+            # TODO: Track actual browser process memory instead of estimates
-                async with LOCK:
+            # These are conservative estimates based on typical Chromium usage
-                    # TODO: Track actual browser process memory instead of estimates
+            permanent_mem = 270 if PERMANENT else 0  # Estimate: ~270MB for permanent browser
-                    # These are conservative estimates based on typical Chromium usage
+            hot_mem = len(HOT_POOL) * 180  # Estimate: ~180MB per hot pool browser
-                    permanent_mem = 270 if PERMANENT else 0  # Estimate: ~270MB for permanent browser
+            cold_mem = len(COLD_POOL) * 180  # Estimate: ~180MB per cold pool browser
-                    hot_mem = len(HOT_POOL) * 180  # Estimate: ~180MB per hot pool browser
+            permanent_active = PERMANENT is not None
-                    cold_mem = len(COLD_POOL) * 180  # Estimate: ~180MB per cold pool browser
+            hot_count = len(HOT_POOL)
-                    permanent_active = PERMANENT is not None
+            cold_count = len(COLD_POOL)
-                    hot_count = len(HOT_POOL)
-                    cold_count = len(COLD_POOL)
-        except asyncio.TimeoutError:
-            logger.warning("Lock acquisition timeout in get_health_summary, using defaults")
-            # Use safe defaults when lock times out
-            permanent_mem = 0
-            hot_mem = 0
-            cold_mem = 0
-            permanent_active = False
-            hot_count = 0
-            cold_count = 0
 
         return {
             "container": {
@@ -561,52 +286,46 @@ class MonitorStats:
         return requests
 
     async def get_browser_list(self) -> List[Dict]:
-        """Get detailed browser pool information with timeout protection."""
+        """Get detailed browser pool information."""
         from crawler_pool import PERMANENT, HOT_POOL, COLD_POOL, LAST_USED, USAGE_COUNT, DEFAULT_CONFIG_SIG, LOCK
 
         browsers = []
         now = time.time()
 
-        # Acquire lock with timeout to prevent deadlock
+        # Acquire lock to prevent race conditions during iteration
-        try:
+        async with LOCK:
-            async with asyncio.timeout(2.0):
+            if PERMANENT:
-                async with LOCK:
+                browsers.append({
-                    if PERMANENT:
+                    "type": "permanent",
-                        browsers.append({
+                    "sig": DEFAULT_CONFIG_SIG[:8] if DEFAULT_CONFIG_SIG else "unknown",
-                            "type": "permanent",
+                    "age_seconds": int(now - self.start_time),
-                            "sig": DEFAULT_CONFIG_SIG[:8] if DEFAULT_CONFIG_SIG else "unknown",
+                    "last_used_seconds": int(now - LAST_USED.get(DEFAULT_CONFIG_SIG, now)),
-                            "age_seconds": int(now - self.start_time),
+                    "memory_mb": 270,
-                            "last_used_seconds": int(now - LAST_USED.get(DEFAULT_CONFIG_SIG, now)),
+                    "hits": USAGE_COUNT.get(DEFAULT_CONFIG_SIG, 0),
-                            "memory_mb": 270,
+                    "killable": False
-                            "hits": USAGE_COUNT.get(DEFAULT_CONFIG_SIG, 0),
+                })
-                            "killable": False
-                        })
 
-                    for sig, crawler in HOT_POOL.items():
+            for sig, crawler in HOT_POOL.items():
-                        browsers.append({
+                browsers.append({
-                            "type": "hot",
+                    "type": "hot",
-                            "sig": sig[:8],
+                    "sig": sig[:8],
-                            "age_seconds": int(now - self.start_time),  # Approximation
+                    "age_seconds": int(now - self.start_time),  # Approximation
-                            "last_used_seconds": int(now - LAST_USED.get(sig, now)),
+                    "last_used_seconds": int(now - LAST_USED.get(sig, now)),
-                            "memory_mb": 180,  # Estimate
+                    "memory_mb": 180,  # Estimate
-                            "hits": USAGE_COUNT.get(sig, 0),
+                    "hits": USAGE_COUNT.get(sig, 0),
-                            "killable": True
+                    "killable": True
-                        })
+                })
 
-                    for sig, crawler in COLD_POOL.items():
+            for sig, crawler in COLD_POOL.items():
-                        browsers.append({
+                browsers.append({
-                            "type": "cold",
+                    "type": "cold",
-                            "sig": sig[:8],
+                    "sig": sig[:8],
-                            "age_seconds": int(now - self.start_time),
+                    "age_seconds": int(now - self.start_time),
-                            "last_used_seconds": int(now - LAST_USED.get(sig, now)),
+                    "last_used_seconds": int(now - LAST_USED.get(sig, now)),
-                            "memory_mb": 180,
+                    "memory_mb": 180,
-                            "hits": USAGE_COUNT.get(sig, 0),
+                    "hits": USAGE_COUNT.get(sig, 0),
-                            "killable": True
+                    "killable": True
-                        })
+                })
-        except asyncio.TimeoutError:
-            logger.error("Browser list lock timeout - pool may be locked by janitor")
-            # Return empty list when lock times out to prevent blocking
-            return []
 
         return browsers
 

deploy/docker/monitor_routes.py

@@ -3,140 +3,14 @@ from fastapi import APIRouter, HTTPException, WebSocket, WebSocketDisconnect
 from pydantic import BaseModel
 from typing import Optional
 from monitor import get_monitor
-from utils import detect_deployment_mode, get_container_id
 import logging
 import asyncio
 import json
-import re
 
 logger = logging.getLogger(__name__)
 router = APIRouter(prefix="/monitor", tags=["monitor"])
 
 
-# ========== Security & Validation ==========
-
-def validate_container_id(cid: str) -> bool:
-    """Validate container ID format to prevent Redis key injection.
-
-    Docker container IDs are 12-64 character hexadecimal strings.
-    Hostnames are alphanumeric with dashes and underscores.
-
-    Args:
-        cid: Container ID to validate
-
-    Returns:
-        True if valid, False otherwise
-    """
-    if not cid or not isinstance(cid, str):
-        return False
-
-    # Allow alphanumeric, dashes, and underscores only (1-64 chars)
-    # This prevents path traversal (../../), wildcards (**), and other injection attempts
-    return bool(re.match(r'^[a-zA-Z0-9_-]{1,64}$', cid))
-
-
-# ========== Redis Aggregation Helpers ==========
-
-async def _get_active_containers():
-    """Get list of active container IDs from Redis with validation."""
-    try:
-        monitor = get_monitor()
-        container_ids = await monitor.redis.smembers("monitor:active_containers")
-
-        # Decode and validate each container ID
-        validated = []
-        for cid in container_ids:
-            cid_str = cid.decode() if isinstance(cid, bytes) else cid
-
-            if validate_container_id(cid_str):
-                validated.append(cid_str)
-            else:
-                logger.warning(f"Invalid container ID format rejected: {cid_str}")
-
-        return validated
-    except Exception as e:
-        logger.error(f"Failed to get active containers: {e}")
-        return []
-
-
-async def _aggregate_active_requests():
-    """Aggregate active requests from all containers."""
-    container_ids = await _get_active_containers()
-    all_requests = []
-
-    monitor = get_monitor()
-    for container_id in container_ids:
-        try:
-            data = await monitor.redis.get(f"monitor:{container_id}:active_requests")
-            if data:
-                requests = json.loads(data)
-                all_requests.extend(requests)
-        except Exception as e:
-            logger.warning(f"Failed to get active requests from {container_id}: {e}")
-
-    return all_requests
-
-
-async def _aggregate_completed_requests(limit=100):
-    """Aggregate completed requests from all containers."""
-    container_ids = await _get_active_containers()
-    all_requests = []
-
-    monitor = get_monitor()
-    for container_id in container_ids:
-        try:
-            data = await monitor.redis.get(f"monitor:{container_id}:completed")
-            if data:
-                requests = json.loads(data)
-                all_requests.extend(requests)
-        except Exception as e:
-            logger.warning(f"Failed to get completed requests from {container_id}: {e}")
-
-    # Sort by end_time (most recent first) and limit
-    all_requests.sort(key=lambda x: x.get("end_time", 0), reverse=True)
-    return all_requests[:limit]
-
-
-async def _aggregate_janitor_events(limit=100):
-    """Aggregate janitor events from all containers."""
-    container_ids = await _get_active_containers()
-    all_events = []
-
-    monitor = get_monitor()
-    for container_id in container_ids:
-        try:
-            data = await monitor.redis.get(f"monitor:{container_id}:janitor")
-            if data:
-                events = json.loads(data)
-                all_events.extend(events)
-        except Exception as e:
-            logger.warning(f"Failed to get janitor events from {container_id}: {e}")
-
-    # Sort by timestamp (most recent first) and limit
-    all_events.sort(key=lambda x: x.get("timestamp", 0), reverse=True)
-    return all_events[:limit]
-
-
-async def _aggregate_errors(limit=100):
-    """Aggregate errors from all containers."""
-    container_ids = await _get_active_containers()
-    all_errors = []
-
-    monitor = get_monitor()
-    for container_id in container_ids:
-        try:
-            data = await monitor.redis.get(f"monitor:{container_id}:errors")
-            if data:
-                errors = json.loads(data)
-                all_errors.extend(errors)
-        except Exception as e:
-            logger.warning(f"Failed to get errors from {container_id}: {e}")
-
-    # Sort by timestamp (most recent first) and limit
-    all_errors.sort(key=lambda x: x.get("timestamp", 0), reverse=True)
-    return all_errors[:limit]
-
-
 @router.get("/health")
 async def get_health():
     """Get current system health snapshot."""
@@ -163,23 +37,18 @@ async def get_requests(status: str = "all", limit: int = 50):
         raise HTTPException(400, f"Invalid limit: {limit}. Must be between 1 and 1000")
 
     try:
-        # Aggregate from all containers via Redis
+        monitor = get_monitor()
-        active_requests = await _aggregate_active_requests()
-        completed_requests = await _aggregate_completed_requests(limit)
-
-        # Filter by status if needed
-        if status in ["success", "error"]:
-            is_success = (status == "success")
-            completed_requests = [r for r in completed_requests if r.get("success") == is_success]
 
         if status == "active":
-            return {"active": active_requests, "completed": []}
+            return {"active": monitor.get_active_requests(), "completed": []}
         elif status == "completed":
-            return {"active": [], "completed": completed_requests}
+            return {"active": [], "completed": monitor.get_completed_requests(limit)}
-        else:  # "all" or success/error
+        elif status in ["success", "error"]:
+            return {"active": [], "completed": monitor.get_completed_requests(limit, status)}
+        else:  # "all"
             return {
-                "active": active_requests,
+                "active": monitor.get_active_requests(),
-                "completed": completed_requests
+                "completed": monitor.get_completed_requests(limit)
             }
     except Exception as e:
         logger.error(f"Error getting requests: {e}")
@@ -191,13 +60,8 @@ async def get_browsers():
     """Get detailed browser pool information."""
     try:
         monitor = get_monitor()
-        container_id = get_container_id()
         browsers = await monitor.get_browser_list()
 
-        # Add container_id to each browser
-        for browser in browsers:
-            browser["container_id"] = container_id
-
         # Calculate summary stats
         total_browsers = len(browsers)
         total_memory = sum(b["memory_mb"] for b in browsers)
@@ -213,8 +77,7 @@ async def get_browsers():
                 "total_count": total_browsers,
                 "total_memory_mb": total_memory,
                 "reuse_rate_percent": round(reuse_rate, 1)
-            },
+            }
-            "container_id": container_id
         }
     except Exception as e:
         logger.error(f"Error getting browsers: {e}")
@@ -262,9 +125,8 @@ async def get_janitor_log(limit: int = 100):
         raise HTTPException(400, f"Invalid limit: {limit}. Must be between 1 and 1000")
 
     try:
-        # Aggregate from all containers via Redis
+        monitor = get_monitor()
-        events = await _aggregate_janitor_events(limit)
+        return {"events": monitor.get_janitor_log(limit)}
-        return {"events": events}
     except Exception as e:
         logger.error(f"Error getting janitor log: {e}")
         raise HTTPException(500, str(e))
@@ -278,9 +140,8 @@ async def get_errors_log(limit: int = 100):
         raise HTTPException(400, f"Invalid limit: {limit}. Must be between 1 and 1000")
 
     try:
-        # Aggregate from all containers via Redis
+        monitor = get_monitor()
-        errors = await _aggregate_errors(limit)
+        return {"errors": monitor.get_errors_log(limit)}
-        return {"errors": errors}
     except Exception as e:
         logger.error(f"Error getting errors log: {e}")
         raise HTTPException(500, str(e))
@@ -489,57 +350,15 @@ async def reset_stats():
         raise HTTPException(500, str(e))
 
 
-@router.get("/containers")
-async def get_containers():
-    """Get container deployment info from Redis heartbeats."""
-    try:
-        monitor = get_monitor()
-        container_ids = await _get_active_containers()
-
-        containers = []
-        for cid in container_ids:
-            try:
-                # Get heartbeat data
-                data = await monitor.redis.get(f"monitor:heartbeat:{cid}")
-                if data:
-                    info = json.loads(data)
-                    containers.append({
-                        "id": info.get("id", cid),
-                        "hostname": info.get("hostname", cid),
-                        "healthy": True  # If heartbeat exists, it's healthy
-                    })
-            except Exception as e:
-                logger.warning(f"Failed to get heartbeat for {cid}: {e}")
-
-        # Determine mode
-        mode = "single" if len(containers) == 1 else "compose"
-        if len(containers) > 1:
-            # Check if any hostname has swarm pattern (service.slot.task_id)
-            if any("." in c["hostname"] and len(c["hostname"].split(".")) > 2 for c in containers):
-                mode = "swarm"
-
-        return {
-            "mode": mode,
-            "container_id": get_container_id(),
-            "containers": containers,
-            "count": len(containers)
-        }
-    except Exception as e:
-        logger.error(f"Error getting containers: {e}")
-        raise HTTPException(500, str(e))
-
-
 @router.websocket("/ws")
 async def websocket_endpoint(websocket: WebSocket):
     """WebSocket endpoint for real-time monitoring updates.
 
-    Sends aggregated updates every 2 seconds from all containers with:
+    Sends updates every 2 seconds with:
-    - Health stats (local container)
+    - Health stats
-    - Active/completed requests (aggregated from all containers)
+    - Active/completed requests
-    - Browser pool status (local container only - not in Redis)
+    - Browser pool status
-    - Timeline data (local container - TODO: aggregate from Redis)
+    - Timeline data
-    - Janitor events (aggregated from all containers)
-    - Errors (aggregated from all containers)
     """
     await websocket.accept()
     logger.info("WebSocket client connected")
@@ -547,46 +366,24 @@ async def websocket_endpoint(websocket: WebSocket):
     try:
         while True:
             try:
-                # Gather aggregated monitoring data from Redis
+                # Gather all monitoring data
                 monitor = get_monitor()
-                container_id = get_container_id()
-
-                # Get container info
-                containers_info = await get_containers()
-
-                # AGGREGATE data from all containers via Redis
-                active_reqs = await _aggregate_active_requests()
-                completed_reqs = await _aggregate_completed_requests(limit=10)
-                janitor_events = await _aggregate_janitor_events(limit=10)
-                errors_log = await _aggregate_errors(limit=10)
-
-                # Local container data (not aggregated)
-                local_health = await monitor.get_health_summary()
-                browsers = await monitor.get_browser_list()  # Browser list is local only
-
-                # Add container_id to browsers (they're local)
-                for browser in browsers:
-                    browser["container_id"] = container_id
 
                 data = {
                     "timestamp": asyncio.get_event_loop().time(),
-                    "container_id": container_id,  # This container handling the WebSocket
+                    "health": await monitor.get_health_summary(),
-                    "is_aggregated": True,  # Flag to indicate aggregated data
-                    "local_health": local_health,  # This container's health
-                    "containers": containers_info.get("containers", []),  # All containers
                     "requests": {
-                        "active": active_reqs,  # Aggregated from all containers
+                        "active": monitor.get_active_requests(),
-                        "completed": completed_reqs  # Aggregated from all containers
+                        "completed": monitor.get_completed_requests(limit=10)
                     },
-                    "browsers": browsers,  # Local only (not in Redis)
+                    "browsers": await monitor.get_browser_list(),
                     "timeline": {
-                        # TODO: Aggregate timeline from Redis (currently local only)
                         "memory": monitor.get_timeline_data("memory", "5m"),
                         "requests": monitor.get_timeline_data("requests", "5m"),
                         "browsers": monitor.get_timeline_data("browsers", "5m")
                     },
-                    "janitor": janitor_events,  # Aggregated from all containers
+                    "janitor": monitor.get_janitor_log(limit=10),
-                    "errors": errors_log  # Aggregated from all containers
+                    "errors": monitor.get_errors_log(limit=10)
                 }
 
                 # Send update to client

deploy/docker/requirements.txt

@@ -12,6 +12,6 @@ pydantic>=2.11
 rank-bm25==0.2.2
 anyio==4.9.0
 PyJWT==2.10.1
-mcp>=1.18.0
+mcp>=1.6.0
 websockets>=15.0.1
 httpx[http2]>=0.27.2

deploy/docker/schemas.py

@@ -1,6 +1,6 @@
 from typing import List, Optional, Dict
 from enum import Enum
-from pydantic import BaseModel, Field, HttpUrl
+from pydantic import BaseModel, Field
 from utils import FilterType
 
 
@@ -85,22 +85,4 @@ class JSEndpointRequest(BaseModel):
     scripts: List[str] = Field(
         ...,
         description="List of separated JavaScript snippets to execute"
-    )
+    )
-
-
-class WebhookConfig(BaseModel):
-    """Configuration for webhook notifications."""
-    webhook_url: HttpUrl
-    webhook_data_in_payload: bool = False
-    webhook_headers: Optional[Dict[str, str]] = None
-
-
-class WebhookPayload(BaseModel):
-    """Payload sent to webhook endpoints."""
-    task_id: str
-    task_type: str  # "crawl", "llm_extraction", etc.
-    status: str  # "completed" or "failed"
-    timestamp: str  # ISO 8601 format
-    urls: List[str]
-    error: Optional[str] = None
-    data: Optional[Dict] = None  # Included only if webhook_data_in_payload=True

deploy/docker/server.py

@@ -200,11 +200,7 @@ async def root():
     return RedirectResponse("/playground")
 
 # ─────────────────── infra / middleware  ─────────────────────
-# Build Redis URL from environment or config
+redis = aioredis.from_url(config["redis"].get("uri", "redis://localhost"))
-redis_host = os.getenv("REDIS_HOST", config["redis"].get("host", "localhost"))
-redis_port = os.getenv("REDIS_PORT", config["redis"].get("port", 6379))
-redis_url = config["redis"].get("uri") or f"redis://{redis_host}:{redis_port}"
-redis = aioredis.from_url(redis_url)
 
 limiter = Limiter(
     key_func=get_remote_address,

deploy/docker/static/monitor/index.html

@@ -116,107 +116,74 @@
 
     <!-- Main Content -->
     <main class="flex-1 overflow-auto p-4 space-y-4">
-        <!-- System Health & Infrastructure (side by side) -->
+        <!-- System Health Bar -->
-        <div class="grid grid-cols-2 gap-4">
+        <section class="bg-surface rounded-lg border border-border p-4">
-            <!-- System Health -->
+            <h2 class="text-sm font-medium mb-3 text-primary">System Health</h2>
-            <section class="bg-surface rounded-lg border border-border p-3">
-                <h2 class="text-sm font-medium mb-2 text-primary">System Health</h2>
 
-                <!-- Row 1: CPU and Memory -->
+            <div class="grid grid-cols-4 gap-4 mb-4">
-                <div class="grid grid-cols-2 gap-3 mb-2">
+                <!-- CPU -->
-                    <!-- CPU -->
+                <div>
-                    <div>
+                    <div class="flex justify-between text-xs mb-1">
-                        <div class="flex justify-between text-xs mb-1">
+                        <span class="text-secondary">CPU</span>
-                            <span class="text-secondary">CPU</span>
+                        <span id="cpu-percent" class="text-light">--%</span>
-                            <span id="cpu-percent" class="text-light">--%</span>
-                        </div>
-                        <div class="w-full bg-dark rounded-full h-2">
-                            <div id="cpu-bar" class="progress-bar h-2 rounded-full bg-primary" style="width: 0%"></div>
-                        </div>
                     </div>
-
+                    <div class="w-full bg-dark rounded-full h-2">
-                    <!-- Memory -->
+                        <div id="cpu-bar" class="progress-bar h-2 rounded-full bg-primary" style="width: 0%"></div>
-                    <div>
-                        <div class="flex justify-between text-xs mb-1">
-                            <span class="text-secondary">Memory</span>
-                            <span id="mem-percent" class="text-light">--%</span>
-                        </div>
-                        <div class="w-full bg-dark rounded-full h-2">
-                            <div id="mem-bar" class="progress-bar h-2 rounded-full bg-accent" style="width: 0%"></div>
-                        </div>
                     </div>
                 </div>
 
-                <!-- Row 2: Network and Uptime -->
+                <!-- Memory -->
-                <div class="grid grid-cols-2 gap-3 mb-2">
+                <div>
-                    <!-- Network -->
+                    <div class="flex justify-between text-xs mb-1">
-                    <div>
+                        <span class="text-secondary">Memory</span>
-                        <div class="flex justify-between text-xs mb-1">
+                        <span id="mem-percent" class="text-light">--%</span>
-                            <span class="text-secondary">Network</span>
-                            <span id="net-io" class="text-light">--</span>
-                        </div>
-                        <div class="text-xs text-secondary">⬆<span id="net-sent">0</span> / ⬇<span id="net-recv">0</span> MB</div>
                     </div>
-
+                    <div class="w-full bg-dark rounded-full h-2">
-                    <!-- Uptime -->
+                        <div id="mem-bar" class="progress-bar h-2 rounded-full bg-accent" style="width: 0%"></div>
-                    <div>
-                        <div class="flex justify-between text-xs mb-1">
-                            <span class="text-secondary">Uptime</span>
-                            <span id="uptime" class="text-light">--</span>
-                        </div>
-                        <div class="text-xs text-secondary" id="last-update">Live: --:--:--</div>
                     </div>
                 </div>
 
-                <!-- Pool Status -->
+                <!-- Network -->
-                <div class="border-t border-border pt-2">
+                <div>
-                    <div class="grid grid-cols-3 gap-3 text-xs">
+                    <div class="flex justify-between text-xs mb-1">
-                        <div>
+                        <span class="text-secondary">Network</span>
-                            <span class="text-secondary">🔥 Permanent:</span>
+                        <span id="net-io" class="text-light">--</span>
-                            <span id="pool-perm" class="text-primary ml-1">INACTIVE (0MB)</span>
-                        </div>
-                        <div>
-                            <span class="text-secondary">♨️ Hot:</span>
-                            <span id="pool-hot" class="text-accent ml-1">0 (0MB)</span>
-                        </div>
-                        <div>
-                            <span class="text-secondary">❄️ Cold:</span>
-                            <span id="pool-cold" class="text-light ml-1">0 (0MB)</span>
-                        </div>
-                    </div>
-                    <div class="mt-1 text-xs text-secondary">
-                        <span>Janitor: </span><span id="janitor-status">adaptive</span> |
-                        <span>Memory pressure: </span><span id="mem-pressure">LOW</span>
                     </div>
+                    <div class="text-xs text-secondary">⬆<span id="net-sent">0</span> MB / ⬇<span id="net-recv">0</span> MB</div>
                 </div>
-            </section>
 
-            <!-- Infrastructure Section -->
+                <!-- Uptime -->
-            <section id="containers-section" class="bg-surface rounded-lg border border-border p-3" style="display: none;">
+                <div>
-            <div class="flex items-center justify-between mb-3">
+                    <div class="flex justify-between text-xs mb-1">
-                <h2 class="text-sm font-medium text-primary">📦 Infrastructure</h2>
+                        <span class="text-secondary">Uptime</span>
-                <div class="flex items-center space-x-2">
+                        <span id="uptime" class="text-light">--</span>
-                    <span class="text-xs text-secondary">Mode:</span>
+                    </div>
-                    <span id="deployment-mode" class="text-xs text-primary font-medium">single</span>
+                    <div class="text-xs text-secondary" id="last-update">Updated: never</div>
-                    <span class="text-xs text-secondary">|</span>
-                    <span class="text-xs text-secondary">Containers:</span>
-                    <span id="container-count" class="text-xs text-accent font-medium">1</span>
                 </div>
             </div>
 
-            <!-- Container Filter Buttons -->
+            <!-- Pool Status -->
-            <div id="container-filters" class="flex flex-wrap gap-2 mb-3">
+            <div class="border-t border-border pt-3">
-                <button class="container-filter-btn px-3 py-1 rounded text-xs bg-primary text-dark font-medium" data-container="all">
+                <div class="grid grid-cols-3 gap-4 text-xs">
-                    All
+                    <div>
-                </button>
+                        <span class="text-secondary">🔥 Permanent:</span>
-            </div>
+                        <span id="pool-perm" class="text-primary ml-2">INACTIVE (0MB)</span>
-
+                    </div>
-            <!-- Container Grid -->
+                    <div>
-            <div id="containers-grid" class="grid grid-cols-3 gap-3 text-xs">
+                        <span class="text-secondary">♨️ Hot:</span>
-                <!-- Containers will be populated here -->
+                        <span id="pool-hot" class="text-accent ml-2">0 (0MB)</span>
+                    </div>
+                    <div>
+                        <span class="text-secondary">❄️ Cold:</span>
+                        <span id="pool-cold" class="text-light ml-2">0 (0MB)</span>
+                    </div>
+                </div>
+                <div class="mt-2 text-xs text-secondary">
+                    <span>Janitor: </span><span id="janitor-status">adaptive</span> |
+                    <span>Memory pressure: </span><span id="mem-pressure">LOW</span>
+                </div>
             </div>
         </section>
-        </div>
 
         <!-- Live Activity Grid (2x2) -->
         <div class="grid grid-cols-2 gap-4">
@@ -256,12 +223,11 @@
                                 <th class="py-1 pr-2">Age</th>
                                 <th class="py-1 pr-2">Used</th>
                                 <th class="py-1 pr-2">Hits</th>
-                                <th class="py-1 pr-2">Container</th>
                                 <th class="py-1">Act</th>
                             </tr>
                         </thead>
                         <tbody id="browsers-table-body">
-                            <tr><td colspan="7" class="text-center py-4 text-secondary">No browsers</td></tr>
+                            <tr><td colspan="6" class="text-center py-4 text-secondary">No browsers</td></tr>
                         </tbody>
                     </table>
                 </div>
@@ -390,16 +356,6 @@
         }
 
         function connectWebSocket() {
-            // Clean up existing connection first to prevent resource leaks
-            if (websocket) {
-                try {
-                    websocket.close();
-                } catch (e) {
-                    console.error('Error closing old WebSocket:', e);
-                }
-                websocket = null;
-            }
-
             if (wsReconnectAttempts >= MAX_WS_RECONNECT) {
                 console.log('Max WebSocket reconnect attempts reached, falling back to polling');
                 useWebSocket = false;
@@ -414,24 +370,9 @@
             const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
             const wsUrl = `${protocol}//${window.location.host}/monitor/ws`;
 
-            try {
+            websocket = new WebSocket(wsUrl);
-                websocket = new WebSocket(wsUrl);
-            } catch (e) {
-                console.error('Failed to create WebSocket:', e);
-                setTimeout(() => connectWebSocket(), 2000 * wsReconnectAttempts);
-                return;
-            }
-
-            // Set connection timeout to prevent indefinite connection attempts
-            const connectionTimeout = setTimeout(() => {
-                if (websocket && websocket.readyState === WebSocket.CONNECTING) {
-                    console.log('WebSocket connection timeout');
-                    websocket.close();
-                }
-            }, 5000);
 
             websocket.onopen = () => {
-                clearTimeout(connectionTimeout);
                 console.log('WebSocket connected');
                 wsReconnectAttempts = 0;
                 updateConnectionStatus('connected');
@@ -444,19 +385,15 @@
             };
 
             websocket.onerror = (error) => {
-                clearTimeout(connectionTimeout);
                 console.error('WebSocket error:', error);
             };
 
-            websocket.onclose = (event) => {
+            websocket.onclose = () => {
-                clearTimeout(connectionTimeout);
+                console.log('WebSocket closed');
-                console.log(`WebSocket closed: code=${event.code}, reason=${event.reason}`);
                 updateConnectionStatus('disconnected', 'Reconnecting...');
 
-                websocket = null;  // Clear reference
+                if (useWebSocket) {
-
+                    setTimeout(connectWebSocket, 2000 * wsReconnectAttempts);
-                if (useWebSocket && wsReconnectAttempts < MAX_WS_RECONNECT) {
-                    setTimeout(() => connectWebSocket(), 2000 * wsReconnectAttempts);
                 } else {
                     startAutoRefresh();
                 }
@@ -522,28 +459,18 @@
         }
 
         function updateRequestsDisplay(requests) {
-            // Filter requests based on current container filter
-            const filteredActive = currentContainerFilter === 'all'
-                ? requests.active
-                : requests.active.filter(r => r.container_id === currentContainerFilter);
-
-            const filteredCompleted = currentContainerFilter === 'all'
-                ? requests.completed
-                : requests.completed.filter(r => r.container_id === currentContainerFilter);
-
             // Update active requests count
             const activeCount = document.getElementById('active-count');
-            if (activeCount) activeCount.textContent = filteredActive.length;
+            if (activeCount) activeCount.textContent = requests.active.length;
 
             // Update active requests list
             const activeList = document.getElementById('active-requests-list');
             if (activeList) {
-                if (filteredActive.length === 0) {
+                if (requests.active.length === 0) {
                     activeList.innerHTML = '<div class="text-secondary text-center py-2">No active requests</div>';
                 } else {
-                    activeList.innerHTML = filteredActive.map(req => `
+                    activeList.innerHTML = requests.active.map(req => `
                         <div class="flex items-center justify-between p-2 bg-dark rounded border border-border">
-                            <span class="text-accent text-xs">${getContainerLabel(req.container_id)}</span>
                             <span class="text-primary">${req.id.substring(0, 8)}</span>
                             <span class="text-secondary">${req.endpoint}</span>
                             <span class="text-light truncate max-w-[200px]" title="${req.url}">${req.url}</span>
@@ -557,12 +484,11 @@
             // Update completed requests
             const completedList = document.getElementById('completed-requests-list');
             if (completedList) {
-                if (filteredCompleted.length === 0) {
+                if (requests.completed.length === 0) {
                     completedList.innerHTML = '<div class="text-secondary text-center py-2">No completed requests</div>';
                 } else {
-                    completedList.innerHTML = filteredCompleted.map(req => `
+                    completedList.innerHTML = requests.completed.map(req => `
                         <div class="flex items-center gap-3 p-2 bg-dark rounded">
-                            <span class="text-accent text-xs w-12 flex-shrink-0">${getContainerLabel(req.container_id)}</span>
                             <span class="text-secondary w-16 flex-shrink-0">${req.id.substring(0, 8)}</span>
                             <span class="text-secondary w-16 flex-shrink-0">${req.endpoint}</span>
                             <span class="text-light truncate flex-1" title="${req.url}">${req.url}</span>
@@ -585,14 +511,6 @@
                         const typeIcon = b.type === 'permanent' ? '🔥' : b.type === 'hot' ? '♨️' : '❄️';
                         const typeColor = b.type === 'permanent' ? 'text-primary' : b.type === 'hot' ? 'text-accent' : 'text-light';
 
-                        // Check if should display based on filter
-                        const shouldDisplay = currentContainerFilter === 'all' ||
-                                            b.container_id === currentContainerFilter;
-                        if (!shouldDisplay) return '';
-
-                        // Find container label (C-1, C-2, etc)
-                        const containerLabel = getContainerLabel(b.container_id);
-
                         return `
                             <tr class="border-t border-border hover:bg-dark">
                                 <td class="py-1 pr-2"><span class="${typeColor}">${typeIcon} ${b.type}</span></td>
@@ -600,7 +518,6 @@
                                 <td class="py-1 pr-2">${formatSeconds(b.age_seconds || 0)}</td>
                                 <td class="py-1 pr-2">${formatSeconds(b.last_used_seconds || 0)}</td>
                                 <td class="py-1 pr-2">${b.hits}</td>
-                                <td class="py-1 pr-2 text-accent text-xs">${containerLabel}</td>
                                 <td class="py-1">
                                     ${b.killable ? `
                                         <button onclick="killBrowser('${b.sig}')" class="text-red-500 hover:underline text-xs">X</button>
@@ -636,23 +553,16 @@
         function updateJanitorDisplay(events) {
             const janitorLog = document.getElementById('janitor-log');
             if (janitorLog) {
-                // Filter events based on current container filter
+                if (events.length === 0) {
-                const filtered = currentContainerFilter === 'all'
-                    ? events
-                    : events.filter(e => e.container_id === currentContainerFilter);
-
-                if (filtered.length === 0) {
                     janitorLog.innerHTML = '<div class="text-secondary text-center py-4">No events yet</div>';
                 } else {
-                    janitorLog.innerHTML = filtered.slice(0, 10).reverse().map(evt => {
+                    janitorLog.innerHTML = events.slice(0, 10).reverse().map(evt => {
                         const time = new Date(evt.timestamp * 1000).toLocaleTimeString();
                         const icon = evt.type === 'close_cold' ? '🧹❄️' : evt.type === 'close_hot' ? '🧹♨️' : '⬆️';
                         const details = JSON.stringify(evt.details);
-                        const containerLabel = getContainerLabel(evt.container_id);
 
                         return `<div class="p-2 bg-dark rounded">
-                            <span class="text-accent text-xs">${containerLabel}</span>
+                            <span class="text-secondary">${time}</span>
-                            <span class="text-secondary ml-2">${time}</span>
                             <span>${icon}</span>
                             <span class="text-primary">${evt.type}</span>
                             <span class="text-secondary">sig=${evt.sig}</span>
@@ -1149,90 +1059,10 @@
             return `${m}m ${s}s`;
         }
 
-        // ========== Containers Management ==========
-        let currentContainerFilter = 'all';
-        let containerMapping = {}; // Maps container_id to label (C-1, C-2, etc)
-
-        // Helper to get container label from ID or hostname
-        function getContainerLabel(containerId) {
-            // Try direct lookup first (works for both hostname and id)
-            if (containerMapping[containerId]) {
-                return containerMapping[containerId];
-            }
-            // Fallback: show first 8 chars of container ID
-            return containerId?.substring(0, 8) || 'unknown';
-        }
-
-        async function fetchContainers() {
-            try {
-                const res = await fetch('/monitor/containers');
-                const data = await res.json();
-
-                document.getElementById('deployment-mode').textContent = data.mode;
-                document.getElementById('container-count').textContent = data.count;
-
-                // Build container ID to label mapping
-                // Use hostname as primary key (friendly name like "crawl4ai-1")
-                // Also map id for backwards compatibility
-                containerMapping = {};
-                data.containers.forEach((c, i) => {
-                    const label = `C-${i+1}`;
-                    containerMapping[c.hostname] = label;  // Map hostname
-                    containerMapping[c.id] = label;  // Also map id
-                });
-
-                // Show section only if multi-container
-                const section = document.getElementById('containers-section');
-                if (data.count > 1) {
-                    section.style.display = 'block';
-
-                    // Update filter buttons
-                    const filtersDiv = document.getElementById('container-filters');
-                    filtersDiv.innerHTML = `
-                        <button class="container-filter-btn px-3 py-1 rounded text-xs ${currentContainerFilter === 'all' ? 'bg-primary text-dark' : 'bg-dark text-secondary'} font-medium" data-container="all">All</button>
-                        ${data.containers.map((c, i) => `
-                            <button class="container-filter-btn px-3 py-1 rounded text-xs ${currentContainerFilter === c.id ? 'bg-primary text-dark' : 'bg-dark text-secondary'}" data-container="${c.id}">C-${i+1}</button>
-                        `).join('')}
-                    `;
-
-                    // Add click handlers to filter buttons
-                    document.querySelectorAll('.container-filter-btn').forEach(btn => {
-                        btn.addEventListener('click', () => {
-                            currentContainerFilter = btn.dataset.container;
-                            fetchContainers(); // Refresh to update button styles
-                            // Re-fetch all data with filter applied
-                            fetchRequests();
-                            fetchBrowsers();
-                            fetchJanitorLogs();
-                            fetchErrorLogs();
-                        });
-                    });
-
-                    // Update containers grid
-                    const grid = document.getElementById('containers-grid');
-                    grid.innerHTML = data.containers.map((c, i) => `
-                        <div class="p-3 bg-dark rounded border ${currentContainerFilter === c.id || currentContainerFilter === 'all' ? 'border-primary' : 'border-border'}">
-                            <div class="flex items-center justify-between mb-2">
-                                <span class="text-primary font-medium">C-${i+1}</span>
-                                <span class="text-xs ${c.healthy ? 'text-accent' : 'text-red-500'}">${c.healthy ? '🟢' : '🔴'}</span>
-                            </div>
-                            <div class="text-xs text-secondary truncate" title="${c.hostname}">${c.hostname}</div>
-                        </div>
-                    `).join('');
-                } else {
-                    section.style.display = 'none';
-                }
-            } catch (e) {
-                console.error('Failed to fetch containers:', e);
-            }
-        }
-
         // ========== Filter change handler ==========
         document.getElementById('filter-requests')?.addEventListener('change', fetchRequests);
 
         // ========== Initialize ==========
-        // Fetch containers info on load
-        fetchContainers();
         // Try WebSocket first, fallback to polling on failure
         connectWebSocket();
     </script>

deploy/docker/tests/cli/README.md

@@ -1,298 +0,0 @@
-# Crawl4AI CLI E2E Test Suite
-
-Comprehensive end-to-end tests for the `crwl server` command-line interface.
-
-## Overview
-
-This test suite validates all aspects of the Docker server CLI including:
-- Basic operations (start, stop, status, logs)
-- Advanced features (scaling, modes, custom configurations)
-- Resource management and stress testing
-- Dashboard UI functionality
-- Edge cases and error handling
-
-**Total Tests:** 32
-- Basic: 8 tests
-- Advanced: 8 tests
-- Resource: 5 tests
-- Dashboard: 1 test
-- Edge Cases: 10 tests
-
-## Prerequisites
-
-```bash
-# Activate virtual environment
-source venv/bin/activate
-
-# For dashboard tests, install Playwright
-pip install playwright
-playwright install chromium
-
-# Ensure Docker is running
-docker ps
-```
-
-## Quick Start
-
-```bash
-# Run all tests (except dashboard)
-./run_tests.sh
-
-# Run specific category
-./run_tests.sh basic
-./run_tests.sh advanced
-./run_tests.sh resource
-./run_tests.sh edge
-
-# Run dashboard tests (slower, includes UI screenshots)
-./run_tests.sh dashboard
-
-# Run specific test
-./run_tests.sh basic 01
-./run_tests.sh edge 05
-```
-
-## Test Categories
-
-### 1. Basic Tests (`basic/`)
-
-Core CLI functionality tests.
-
-| Test | Description | Expected Result |
-|------|-------------|----------------|
-| `test_01_start_default.sh` | Start server with defaults | 1 replica on port 11235 |
-| `test_02_status.sh` | Check server status | Shows running state and details |
-| `test_03_stop.sh` | Stop server | Clean shutdown, port freed |
-| `test_04_start_custom_port.sh` | Start on port 8080 | Server on custom port |
-| `test_05_start_replicas.sh` | Start with 3 replicas | Multi-container deployment |
-| `test_06_logs.sh` | View server logs | Logs displayed correctly |
-| `test_07_restart.sh` | Restart server | Preserves configuration |
-| `test_08_cleanup.sh` | Force cleanup | All resources removed |
-
-### 2. Advanced Tests (`advanced/`)
-
-Advanced features and configurations.
-
-| Test | Description | Expected Result |
-|------|-------------|----------------|
-| `test_01_scale_up.sh` | Scale 3 → 5 replicas | Live scaling without downtime |
-| `test_02_scale_down.sh` | Scale 5 → 2 replicas | Graceful container removal |
-| `test_03_mode_single.sh` | Explicit single mode | Single container deployment |
-| `test_04_mode_compose.sh` | Compose mode with Nginx | Multi-container with load balancer |
-| `test_05_custom_image.sh` | Custom image specification | Uses specified image tag |
-| `test_06_env_file.sh` | Environment file loading | Variables loaded correctly |
-| `test_07_stop_remove_volumes.sh` | Stop with volume removal | Volumes cleaned up |
-| `test_08_restart_with_scale.sh` | Restart with new replica count | Configuration updated |
-
-### 3. Resource Tests (`resource/`)
-
-Resource monitoring and stress testing.
-
-| Test | Description | Expected Result |
-|------|-------------|----------------|
-| `test_01_memory_monitoring.sh` | Monitor memory usage | Stats accessible and reasonable |
-| `test_02_cpu_stress.sh` | Concurrent request load | Handles load without errors |
-| `test_03_max_replicas.sh` | 10 replicas stress test | Maximum scale works correctly |
-| `test_04_cleanup_verification.sh` | Verify resource cleanup | All Docker resources removed |
-| `test_05_long_running.sh` | 5-minute stability test | Server remains stable |
-
-### 4. Dashboard Tests (`dashboard/`)
-
-Dashboard UI functionality with Playwright.
-
-| Test | Description | Expected Result |
-|------|-------------|----------------|
-| `test_01_dashboard_ui.py` | Full dashboard UI test | All UI elements functional |
-
-**Dashboard Test Details:**
-- Starts server with 3 replicas
-- Runs demo script to generate activity
-- Uses Playwright to:
-  - Take screenshots of dashboard
-  - Verify container filter buttons
-  - Check WebSocket connection
-  - Validate timeline charts
-  - Test all dashboard sections
-
-**Screenshots saved to:** `dashboard/screenshots/`
-
-### 5. Edge Case Tests (`edge/`)
-
-Error handling and validation.
-
-| Test | Description | Expected Result |
-|------|-------------|----------------|
-| `test_01_already_running.sh` | Start when already running | Proper error message |
-| `test_02_not_running.sh` | Operations when stopped | Appropriate errors |
-| `test_03_scale_single_mode.sh` | Scale single container | Error with guidance |
-| `test_04_invalid_port.sh` | Invalid port numbers | Validation errors |
-| `test_05_invalid_replicas.sh` | Invalid replica counts | Validation errors |
-| `test_06_missing_env_file.sh` | Non-existent env file | File not found error |
-| `test_07_port_in_use.sh` | Port already occupied | Port conflict error |
-| `test_08_state_corruption.sh` | Corrupted state file | Cleanup recovers |
-| `test_09_network_conflict.sh` | Docker network collision | Handles gracefully |
-| `test_10_rapid_operations.sh` | Rapid start/stop cycles | No corruption |
-
-## Test Execution Workflow
-
-Each test follows this pattern:
-
-1. **Setup:** Clean state, activate venv
-2. **Execute:** Run test commands
-3. **Verify:** Check results and assertions
-4. **Cleanup:** Stop server, remove resources
-
-## Running Individual Tests
-
-```bash
-# Make test executable (if needed)
-chmod +x deploy/docker/tests/cli/basic/test_01_start_default.sh
-
-# Run directly
-./deploy/docker/tests/cli/basic/test_01_start_default.sh
-
-# Or use the test runner
-./run_tests.sh basic 01
-```
-
-## Interpreting Results
-
-### Success Output
-```
-✅ Test passed: [description]
-```
-
-### Failure Output
-```
-❌ Test failed: [error message]
-```
-
-### Warning Output
-```
-⚠️  Warning: [issue description]
-```
-
-## Common Issues
-
-### Docker Not Running
-```
-Error: Docker daemon not running
-Solution: Start Docker Desktop or Docker daemon
-```
-
-### Port Already In Use
-```
-Error: Port 11235 is already in use
-Solution: Stop existing server or use different port
-```
-
-### Virtual Environment Not Found
-```
-Warning: venv not found
-Solution: Create venv and activate it
-```
-
-### Playwright Not Installed
-```
-Error: playwright module not found
-Solution: pip install playwright && playwright install chromium
-```
-
-## Test Development
-
-### Adding New Tests
-
-1. **Choose category:** basic, advanced, resource, dashboard, or edge
-2. **Create test file:** Follow naming pattern `test_XX_description.sh`
-3. **Use template:**
-
-```bash
-#!/bin/bash
-# Test: [Description]
-# Expected: [What should happen]
-
-set -e
-
-echo "=== Test: [Name] ==="
-echo ""
-
-source venv/bin/activate
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Test logic here
-
-# Cleanup
-crwl server stop >/dev/null 2>&1
-
-echo ""
-echo "✅ Test passed: [success message]"
-```
-
-4. **Make executable:** `chmod +x test_XX_description.sh`
-5. **Test it:** `./test_XX_description.sh`
-6. **Add to runner:** Tests are auto-discovered by `run_tests.sh`
-
-## CI/CD Integration
-
-These tests can be integrated into CI/CD pipelines:
-
-```yaml
-# Example GitHub Actions
-- name: Run CLI Tests
-  run: |
-    source venv/bin/activate
-    cd deploy/docker/tests/cli
-    ./run_tests.sh all
-```
-
-## Performance Considerations
-
-- **Basic tests:** ~2-5 minutes total
-- **Advanced tests:** ~5-10 minutes total
-- **Resource tests:** ~10-15 minutes total (including 5-min stability test)
-- **Dashboard test:** ~3-5 minutes
-- **Edge case tests:** ~5-8 minutes total
-
-**Full suite:** ~30-45 minutes
-
-## Best Practices
-
-1. **Always cleanup:** Each test should cleanup after itself
-2. **Wait for readiness:** Add sleep after starting servers
-3. **Check health:** Verify health endpoint before assertions
-4. **Graceful failures:** Use `|| true` to continue on expected failures
-5. **Clear messages:** Output should clearly indicate what's being tested
-
-## Troubleshooting
-
-### Tests Hanging
-- Check if Docker containers are stuck
-- Look for port conflicts
-- Verify network connectivity
-
-### Intermittent Failures
-- Increase sleep durations for slower systems
-- Check system resources (memory, CPU)
-- Verify Docker has enough resources allocated
-
-### All Tests Failing
-- Verify Docker is running: `docker ps`
-- Check CLI is installed: `which crwl`
-- Activate venv: `source venv/bin/activate`
-- Check server manager: `crwl server status`
-
-## Contributing
-
-When adding new tests:
-1. Follow existing naming conventions
-2. Add comprehensive documentation
-3. Test on clean system
-4. Update this README
-5. Ensure cleanup is robust
-
-## License
-
-Same as Crawl4AI project license.

deploy/docker/tests/cli/TEST_RESULTS.md

@@ -1,163 +0,0 @@
-# CLI Test Suite - Execution Results
-
-**Date:** 2025-10-20
-**Status:** ✅ PASSED
-
-## Summary
-
-| Category | Total | Passed | Failed | Skipped |
-|----------|-------|--------|--------|---------|
-| Basic Tests | 8 | 8 | 0 | 0 |
-| Advanced Tests | 8 | 8 | 0 | 0 |
-| Edge Case Tests | 10 | 10 | 0 | 0 |
-| Resource Tests | 3 | 3 | 0 | 2 (skipped) |
-| Dashboard UI Tests | 0 | 0 | 0 | 1 (not run) |
-| **TOTAL** | **29** | **29** | **0** | **3** |
-
-**Success Rate:** 100% (29/29 tests passed)
-
-## Test Results by Category
-
-### ✅ Basic Tests (8/8 Passed)
-
-| Test | Status | Notes |
-|------|--------|-------|
-| test_01_start_default | ✅ PASS | Server starts with defaults (1 replica, port 11235) |
-| test_02_status | ✅ PASS | Status command shows correct information |
-| test_03_stop | ✅ PASS | Server stops cleanly, port freed |
-| test_04_start_custom_port | ✅ PASS | Server starts on port 8080 |
-| test_05_start_replicas | ✅ PASS | Compose mode with 3 replicas |
-| test_06_logs | ✅ PASS | Logs retrieved successfully |
-| test_07_restart | ✅ PASS | Server restarts preserving config (2 replicas) |
-| test_08_cleanup | ✅ PASS | Force cleanup removes all resources |
-
-### ✅ Advanced Tests (8/8 Passed)
-
-| Test | Status | Notes |
-|------|--------|-------|
-| test_01_scale_up | ✅ PASS | Scaled 3 → 5 replicas successfully |
-| test_02_scale_down | ✅ PASS | Scaled 5 → 2 replicas successfully |
-| test_03_mode_single | ✅ PASS | Explicit single mode works |
-| test_04_mode_compose | ✅ PASS | Compose mode with 3 replicas and Nginx |
-| test_05_custom_image | ✅ PASS | Custom image specification works |
-| test_06_env_file | ✅ PASS | Environment file loading works |
-| test_07_stop_remove_volumes | ✅ PASS | Volumes handled during cleanup |
-| test_08_restart_with_scale | ✅ PASS | Restart with scale change (2 → 4 replicas) |
-
-### ✅ Edge Case Tests (10/10 Passed)
-
-| Test | Status | Notes |
-|------|--------|-------|
-| test_01_already_running | ✅ PASS | Proper error for duplicate start |
-| test_02_not_running | ✅ PASS | Appropriate errors when server stopped |
-| test_03_scale_single_mode | ✅ PASS | Cannot scale single mode (expected error) |
-| test_04_invalid_port | ✅ PASS | Rejected ports: 0, -1, 99999, 65536 |
-| test_05_invalid_replicas | ✅ PASS | Rejected replicas: 0, -1, 101 |
-| test_06_missing_env_file | ✅ PASS | File not found error |
-| test_07_port_in_use | ✅ PASS | Port conflict detected |
-| test_08_state_corruption | ✅ PASS | Corrupted state handled gracefully |
-| test_09_network_conflict | ✅ PASS | Network collision handled |
-| test_10_rapid_operations | ✅ PASS | Rapid start/stop/restart cycles work |
-
-### ✅ Resource Tests (3/5 Completed)
-
-| Test | Status | Notes |
-|------|--------|-------|
-| test_01_memory_monitoring | ✅ PASS | Baseline: 9.6%, After: 12.1%, Pool: 450 MB |
-| test_02_cpu_stress | ✅ PASS | Handled 10 concurrent requests |
-| test_03_max_replicas | ⏭️ SKIP | Takes ~2 minutes (10 replicas) |
-| test_04_cleanup_verification | ✅ PASS | All resources cleaned up |
-| test_05_long_running | ⏭️ SKIP | Takes 5 minutes |
-
-### Dashboard UI Tests (Not Run)
-
-| Test | Status | Notes |
-|------|--------|-------|
-| test_01_dashboard_ui | ⏭️ SKIP | Requires Playwright, takes ~5 minutes |
-
-## Key Findings
-
-### ✅ Strengths
-
-1. **Robust Error Handling**
-   - All invalid inputs properly rejected with clear error messages
-   - State corruption detected and recovered automatically
-   - Port conflicts identified before container start
-
-2. **Scaling Functionality**
-   - Live scaling works smoothly (3 → 5 → 2 replicas)
-   - Mode detection works correctly (single vs compose)
-   - Restart preserves configuration
-
-3. **Resource Management**
-   - Cleanup thoroughly removes all Docker resources
-   - Memory usage reasonable (9.6% → 12.1% with 5 crawls)
-   - Concurrent requests handled without errors
-
-4. **CLI Usability**
-   - Clear, color-coded output
-   - Helpful error messages with hints
-   - Status command shows comprehensive info
-
-### 📊 Performance Observations
-
-- **Startup Time:** ~5 seconds for single container, ~10-12 seconds for 3 replicas
-- **Memory Usage:** Baseline 9.6%, increases to 12.1% after 5 crawls
-- **Browser Pool:** ~450 MB memory usage (reasonable)
-- **Concurrent Load:** Successfully handled 10 parallel requests
-
-### 🔧 Issues Found
-
-None! All 29 tests passed successfully.
-
-## Test Execution Notes
-
-### Test Environment
-- **OS:** macOS (Darwin 24.3.0)
-- **Docker:** Running
-- **Python:** Virtual environment activated
-- **Date:** 2025-10-20
-
-### Skipped Tests Rationale
-1. **test_03_max_replicas:** Takes ~2 minutes to start 10 replicas
-2. **test_05_long_running:** 5-minute stability test
-3. **test_01_dashboard_ui:** Requires Playwright installation, UI screenshots
-
-These tests are fully implemented and can be run manually when time permits.
-
-## Verification Commands
-
-All tests can be re-run with:
-
-```bash
-# Individual test
-bash deploy/docker/tests/cli/basic/test_01_start_default.sh
-
-# Category
-./deploy/docker/tests/cli/run_tests.sh basic
-
-# All tests
-./deploy/docker/tests/cli/run_tests.sh all
-```
-
-## Conclusion
-
-✅ **The CLI test suite is comprehensive and thoroughly validates all functionality.**
-
-- All core features tested and working
-- Error handling is robust
-- Edge cases properly covered
-- Resource management verified
-- No bugs or issues found
-
-The Crawl4AI Docker server CLI is production-ready with excellent test coverage.
-
----
-
-**Next Steps:**
-1. Run skipped tests when time permits (optional)
-2. Integrate into CI/CD pipeline
-3. Run dashboard UI test for visual verification
-4. Document test results in main README
-
-**Recommendation:** ✅ Ready for production use

deploy/docker/tests/cli/TEST_SUMMARY.md

@@ -1,300 +0,0 @@
-# CLI Test Suite - Implementation Summary
-
-## Completed Implementation
-
-Successfully created a comprehensive E2E test suite for the Crawl4AI Docker server CLI.
-
-## Test Suite Overview
-
-### Total Tests: 32
-
-#### 1. Basic Tests (8 tests) ✅
-- `test_01_start_default.sh` - Start with default settings
-- `test_02_status.sh` - Status command validation
-- `test_03_stop.sh` - Clean server shutdown
-- `test_04_start_custom_port.sh` - Custom port configuration
-- `test_05_start_replicas.sh` - Multi-replica deployment
-- `test_06_logs.sh` - Log retrieval
-- `test_07_restart.sh` - Server restart
-- `test_08_cleanup.sh` - Force cleanup
-
-#### 2. Advanced Tests (8 tests) ✅
-- `test_01_scale_up.sh` - Scale from 3 to 5 replicas
-- `test_02_scale_down.sh` - Scale from 5 to 2 replicas
-- `test_03_mode_single.sh` - Explicit single mode
-- `test_04_mode_compose.sh` - Compose mode with Nginx
-- `test_05_custom_image.sh` - Custom image specification
-- `test_06_env_file.sh` - Environment file loading
-- `test_07_stop_remove_volumes.sh` - Volume cleanup
-- `test_08_restart_with_scale.sh` - Restart with scale change
-
-#### 3. Resource Tests (5 tests) ✅
-- `test_01_memory_monitoring.sh` - Memory usage tracking
-- `test_02_cpu_stress.sh` - CPU stress with concurrent requests
-- `test_03_max_replicas.sh` - Maximum (10) replicas stress test
-- `test_04_cleanup_verification.sh` - Resource cleanup verification
-- `test_05_long_running.sh` - 5-minute stability test
-
-#### 4. Dashboard UI Test (1 test) ✅
-- `test_01_dashboard_ui.py` - Comprehensive Playwright test
-  - Automated browser testing
-  - Screenshot capture (7 screenshots per run)
-  - UI element validation
-  - Container filter testing
-  - WebSocket connection verification
-
-#### 5. Edge Case Tests (10 tests) ✅
-- `test_01_already_running.sh` - Duplicate start attempt
-- `test_02_not_running.sh` - Operations on stopped server
-- `test_03_scale_single_mode.sh` - Invalid scaling operation
-- `test_04_invalid_port.sh` - Port validation (0, -1, 99999, 65536)
-- `test_05_invalid_replicas.sh` - Replica validation (0, -1, 101)
-- `test_06_missing_env_file.sh` - Non-existent env file
-- `test_07_port_in_use.sh` - Port conflict detection
-- `test_08_state_corruption.sh` - State file corruption recovery
-- `test_09_network_conflict.sh` - Docker network collision handling
-- `test_10_rapid_operations.sh` - Rapid start/stop cycles
-
-## Test Infrastructure
-
-### Master Test Runner (`run_tests.sh`)
-- Run all tests or specific categories
-- Color-coded output (green/red/yellow)
-- Test counters (passed/failed/skipped)
-- Summary statistics
-- Individual test execution support
-
-### Documentation
-- `README.md` - Comprehensive test documentation
-  - Test descriptions and expected results
-  - Usage instructions
-  - Troubleshooting guide
-  - Best practices
-  - CI/CD integration examples
-
-- `TEST_SUMMARY.md` - Implementation summary (this file)
-
-## File Structure
-
-```
-deploy/docker/tests/cli/
-├── README.md                      # Main documentation
-├── TEST_SUMMARY.md                # This summary
-├── run_tests.sh                   # Master test runner
-│
-├── basic/                         # Basic CLI tests
-│   ├── test_01_start_default.sh
-│   ├── test_02_status.sh
-│   ├── test_03_stop.sh
-│   ├── test_04_start_custom_port.sh
-│   ├── test_05_start_replicas.sh
-│   ├── test_06_logs.sh
-│   ├── test_07_restart.sh
-│   └── test_08_cleanup.sh
-│
-├── advanced/                      # Advanced feature tests
-│   ├── test_01_scale_up.sh
-│   ├── test_02_scale_down.sh
-│   ├── test_03_mode_single.sh
-│   ├── test_04_mode_compose.sh
-│   ├── test_05_custom_image.sh
-│   ├── test_06_env_file.sh
-│   ├── test_07_stop_remove_volumes.sh
-│   └── test_08_restart_with_scale.sh
-│
-├── resource/                      # Resource and stress tests
-│   ├── test_01_memory_monitoring.sh
-│   ├── test_02_cpu_stress.sh
-│   ├── test_03_max_replicas.sh
-│   ├── test_04_cleanup_verification.sh
-│   └── test_05_long_running.sh
-│
-├── dashboard/                     # Dashboard UI tests
-│   ├── test_01_dashboard_ui.py
-│   ├── run_dashboard_test.sh
-│   └── screenshots/               # Auto-generated screenshots
-│
-└── edge/                          # Edge case tests
-    ├── test_01_already_running.sh
-    ├── test_02_not_running.sh
-    ├── test_03_scale_single_mode.sh
-    ├── test_04_invalid_port.sh
-    ├── test_05_invalid_replicas.sh
-    ├── test_06_missing_env_file.sh
-    ├── test_07_port_in_use.sh
-    ├── test_08_state_corruption.sh
-    ├── test_09_network_conflict.sh
-    └── test_10_rapid_operations.sh
-```
-
-## Usage Examples
-
-### Run All Tests (except dashboard)
-```bash
-./run_tests.sh
-```
-
-### Run Specific Category
-```bash
-./run_tests.sh basic
-./run_tests.sh advanced
-./run_tests.sh resource
-./run_tests.sh edge
-```
-
-### Run Dashboard Tests
-```bash
-./run_tests.sh dashboard
-# or
-./dashboard/run_dashboard_test.sh
-```
-
-### Run Individual Test
-```bash
-./run_tests.sh basic 01
-./run_tests.sh edge 05
-```
-
-### Direct Execution
-```bash
-./basic/test_01_start_default.sh
-./edge/test_01_already_running.sh
-```
-
-## Test Verification
-
-The following tests have been verified working:
-- ✅ `test_01_start_default.sh` - PASSED
-- ✅ `test_02_status.sh` - PASSED
-- ✅ `test_03_stop.sh` - PASSED
-- ✅ `test_03_mode_single.sh` - PASSED
-- ✅ `test_01_already_running.sh` - PASSED
-- ✅ Master test runner - PASSED
-
-## Key Features
-
-### Robustness
-- Each test cleans up after itself
-- Handles expected failures gracefully
-- Waits for server readiness before assertions
-- Comprehensive error checking
-
-### Clarity
-- Clear test descriptions
-- Colored output for easy interpretation
-- Detailed error messages
-- Progress indicators
-
-### Completeness
-- Covers all CLI commands
-- Tests success and failure paths
-- Validates error messages
-- Checks resource cleanup
-
-### Maintainability
-- Consistent structure across all tests
-- Well-documented code
-- Modular test design
-- Easy to add new tests
-
-## Test Coverage
-
-### CLI Commands Tested
-- ✅ `crwl server start` (all options)
-- ✅ `crwl server stop` (with/without volumes)
-- ✅ `crwl server status`
-- ✅ `crwl server scale`
-- ✅ `crwl server logs`
-- ✅ `crwl server restart`
-- ✅ `crwl server cleanup`
-
-### Deployment Modes Tested
-- ✅ Single container mode
-- ✅ Compose mode (multi-container)
-- ✅ Auto mode detection
-
-### Features Tested
-- ✅ Custom ports
-- ✅ Custom replicas (1-10)
-- ✅ Custom images
-- ✅ Environment files
-- ✅ Live scaling
-- ✅ Configuration persistence
-- ✅ Resource cleanup
-- ✅ Dashboard UI
-
-### Error Handling Tested
-- ✅ Invalid inputs (ports, replicas)
-- ✅ Missing files
-- ✅ Port conflicts
-- ✅ State corruption
-- ✅ Network conflicts
-- ✅ Rapid operations
-- ✅ Duplicate operations
-
-## Performance
-
-### Estimated Execution Times
-- Basic tests: ~2-5 minutes
-- Advanced tests: ~5-10 minutes
-- Resource tests: ~10-15 minutes
-- Dashboard test: ~3-5 minutes
-- Edge case tests: ~5-8 minutes
-
-**Total: ~30-45 minutes for full suite**
-
-## Next Steps
-
-### Recommended Actions
-1. ✅ Run full test suite to verify all tests
-2. ✅ Test dashboard UI test with Playwright
-3. ✅ Verify long-running stability test
-4. ✅ Integrate into CI/CD pipeline
-5. ✅ Add to project documentation
-
-### Future Enhancements
-- Add performance benchmarking
-- Add load testing scenarios
-- Add network failure simulation
-- Add disk space tests
-- Add security tests
-- Add multi-host tests (Swarm mode)
-
-## Notes
-
-### Dependencies
-- Docker running
-- Virtual environment activated
-- `jq` for JSON parsing (installed by default on most systems)
-- `bc` for calculations (installed by default on most systems)
-- Playwright for dashboard tests (optional)
-
-### Test Philosophy
-- **Small:** Each test focuses on one specific aspect
-- **Smart:** Tests verify both success and failure paths
-- **Strong:** Robust cleanup and error handling
-- **Self-contained:** Each test is independent
-
-### Known Limitations
-- Dashboard test requires Playwright installation
-- Long-running test takes 5 minutes
-- Max replicas test requires significant system resources
-- Some tests may need adjustment for slower systems
-
-## Success Criteria
-
-✅ All 32 tests created
-✅ Test runner implemented
-✅ Documentation complete
-✅ Tests verified working
-✅ File structure organized
-✅ Error handling comprehensive
-✅ Cleanup mechanisms robust
-
-## Conclusion
-
-The CLI test suite is complete and ready for use. It provides comprehensive coverage of all CLI functionality, validates error handling, and ensures robustness across various scenarios.
-
-**Status:** ✅ COMPLETE
-**Date:** 2025-10-20
-**Tests:** 32 (8 basic + 8 advanced + 5 resource + 1 dashboard + 10 edge)

deploy/docker/tests/cli/advanced/test_01_scale_up.sh

@@ -1,56 +0,0 @@
-#!/bin/bash
-# Test: Scale server up from 3 to 5 replicas
-# Expected: Server scales without downtime
-
-set -e
-
-echo "=== Test: Scale Up (3 → 5 replicas) ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start with 3 replicas
-echo "Starting server with 3 replicas..."
-crwl server start --replicas 3 >/dev/null 2>&1
-sleep 10
-
-# Verify 3 replicas
-STATUS=$(crwl server status | grep "Replicas" || echo "")
-echo "Initial status: $STATUS"
-
-# Scale up to 5
-echo ""
-echo "Scaling up to 5 replicas..."
-crwl server scale 5
-
-sleep 10
-
-# Verify 5 replicas
-STATUS=$(crwl server status)
-echo "$STATUS"
-
-if ! echo "$STATUS" | grep -q "5"; then
-    echo "❌ Status does not show 5 replicas"
-    crwl server stop
-    exit 1
-fi
-
-# Verify health during scaling
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "❌ Health check failed after scaling"
-    crwl server stop
-    exit 1
-fi
-
-# Cleanup
-echo "Cleaning up..."
-crwl server stop >/dev/null 2>&1
-
-echo ""
-echo "✅ Test passed: Successfully scaled from 3 to 5 replicas"

deploy/docker/tests/cli/advanced/test_02_scale_down.sh

@@ -1,56 +0,0 @@
-#!/bin/bash
-# Test: Scale server down from 5 to 2 replicas
-# Expected: Server scales down gracefully
-
-set -e
-
-echo "=== Test: Scale Down (5 → 2 replicas) ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start with 5 replicas
-echo "Starting server with 5 replicas..."
-crwl server start --replicas 5 >/dev/null 2>&1
-sleep 12
-
-# Verify 5 replicas
-STATUS=$(crwl server status | grep "Replicas" || echo "")
-echo "Initial status: $STATUS"
-
-# Scale down to 2
-echo ""
-echo "Scaling down to 2 replicas..."
-crwl server scale 2
-
-sleep 8
-
-# Verify 2 replicas
-STATUS=$(crwl server status)
-echo "$STATUS"
-
-if ! echo "$STATUS" | grep -q "2"; then
-    echo "❌ Status does not show 2 replicas"
-    crwl server stop
-    exit 1
-fi
-
-# Verify health after scaling down
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "❌ Health check failed after scaling down"
-    crwl server stop
-    exit 1
-fi
-
-# Cleanup
-echo "Cleaning up..."
-crwl server stop >/dev/null 2>&1
-
-echo ""
-echo "✅ Test passed: Successfully scaled down from 5 to 2 replicas"

deploy/docker/tests/cli/advanced/test_03_mode_single.sh

@@ -1,52 +0,0 @@
-#!/bin/bash
-# Test: Start server explicitly in single mode
-# Expected: Server starts in single mode
-
-set -e
-
-echo "=== Test: Explicit Single Mode ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start in single mode explicitly
-echo "Starting server in single mode..."
-crwl server start --mode single
-
-sleep 5
-
-# Check mode
-STATUS=$(crwl server status)
-echo "$STATUS"
-
-if ! echo "$STATUS" | grep -q "single"; then
-    echo "❌ Mode is not 'single'"
-    crwl server stop
-    exit 1
-fi
-
-if ! echo "$STATUS" | grep -q "1"; then
-    echo "❌ Should have 1 replica in single mode"
-    crwl server stop
-    exit 1
-fi
-
-# Verify health
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "❌ Health check failed"
-    crwl server stop
-    exit 1
-fi
-
-# Cleanup
-echo "Cleaning up..."
-crwl server stop >/dev/null 2>&1
-
-echo ""
-echo "✅ Test passed: Server started in single mode"

deploy/docker/tests/cli/advanced/test_04_mode_compose.sh

@@ -1,52 +0,0 @@
-#!/bin/bash
-# Test: Start server in compose mode with replicas
-# Expected: Server starts in compose mode with Nginx
-
-set -e
-
-echo "=== Test: Compose Mode with 3 Replicas ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start in compose mode
-echo "Starting server in compose mode with 3 replicas..."
-crwl server start --mode compose --replicas 3
-
-sleep 12
-
-# Check mode
-STATUS=$(crwl server status)
-echo "$STATUS"
-
-if ! echo "$STATUS" | grep -q "3"; then
-    echo "❌ Status does not show 3 replicas"
-    crwl server stop
-    exit 1
-fi
-
-# Verify Nginx is running (load balancer)
-NGINX_RUNNING=$(docker ps --filter "name=nginx" --format "{{.Names}}" || echo "")
-if [[ -z "$NGINX_RUNNING" ]]; then
-    echo "⚠️  Warning: Nginx load balancer not detected (may be using swarm or single mode)"
-fi
-
-# Verify health through load balancer
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "❌ Health check failed"
-    crwl server stop
-    exit 1
-fi
-
-# Cleanup
-echo "Cleaning up..."
-crwl server stop >/dev/null 2>&1
-
-echo ""
-echo "✅ Test passed: Server started in compose mode"

deploy/docker/tests/cli/advanced/test_05_custom_image.sh

@@ -1,47 +0,0 @@
-#!/bin/bash
-# Test: Start server with custom image tag
-# Expected: Server uses specified image
-
-set -e
-
-echo "=== Test: Custom Image Specification ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Use latest tag explicitly (or specify a different tag if available)
-IMAGE="unclecode/crawl4ai:latest"
-echo "Starting server with image: $IMAGE..."
-crwl server start --image "$IMAGE"
-
-sleep 5
-
-# Check status shows correct image
-STATUS=$(crwl server status)
-echo "$STATUS"
-
-if ! echo "$STATUS" | grep -q "crawl4ai"; then
-    echo "❌ Status does not show correct image"
-    crwl server stop
-    exit 1
-fi
-
-# Verify health
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "❌ Health check failed"
-    crwl server stop
-    exit 1
-fi
-
-# Cleanup
-echo "Cleaning up..."
-crwl server stop >/dev/null 2>&1
-
-echo ""
-echo "✅ Test passed: Server started with custom image"

deploy/docker/tests/cli/advanced/test_06_env_file.sh

@@ -1,47 +0,0 @@
-#!/bin/bash
-# Test: Start server with environment file
-# Expected: Server loads environment variables
-
-set -e
-
-echo "=== Test: Start with Environment File ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Create a test env file
-TEST_ENV_FILE="/tmp/test_crawl4ai.env"
-cat > "$TEST_ENV_FILE" <<EOF
-TEST_VAR=test_value
-OPENAI_API_KEY=sk-test-key
-EOF
-
-echo "Created test env file at $TEST_ENV_FILE"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start with env file
-echo "Starting server with env file..."
-crwl server start --env-file "$TEST_ENV_FILE"
-
-sleep 5
-
-# Verify server started
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "❌ Health check failed"
-    rm -f "$TEST_ENV_FILE"
-    crwl server stop
-    exit 1
-fi
-
-# Cleanup
-echo "Cleaning up..."
-crwl server stop >/dev/null 2>&1
-rm -f "$TEST_ENV_FILE"
-
-echo ""
-echo "✅ Test passed: Server started with environment file"

deploy/docker/tests/cli/advanced/test_07_stop_remove_volumes.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-# Test: Stop server with volume removal
-# Expected: Volumes are removed along with containers
-
-set -e
-
-echo "=== Test: Stop with Remove Volumes ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Start server (which may create volumes)
-echo "Starting server..."
-crwl server start --replicas 2 >/dev/null 2>&1
-sleep 8
-
-# Make some requests to populate data
-echo "Making requests to populate data..."
-curl -s -X POST http://localhost:11235/crawl \
-  -H "Content-Type: application/json" \
-  -d '{"urls": ["https://httpbin.org/html"], "crawler_config": {}}' > /dev/null || true
-
-sleep 2
-
-# Stop with volume removal (needs confirmation, so we'll use cleanup instead)
-echo "Stopping server with volume removal..."
-# Note: --remove-volumes requires confirmation, so we use cleanup --force
-crwl server cleanup --force >/dev/null 2>&1
-
-sleep 3
-
-# Verify volumes are removed
-echo "Checking for remaining volumes..."
-VOLUMES=$(docker volume ls --filter "name=crawl4ai" --format "{{.Name}}" || echo "")
-if [[ -n "$VOLUMES" ]]; then
-    echo "⚠️  Warning: Some volumes still exist: $VOLUMES"
-    echo "  (This may be expected if using system-wide volumes)"
-fi
-
-# Verify server is stopped
-STATUS=$(crwl server status | grep "No server" || echo "RUNNING")
-if [[ "$STATUS" == "RUNNING" ]]; then
-    echo "❌ Server still running after stop"
-    exit 1
-fi
-
-echo ""
-echo "✅ Test passed: Server stopped and volumes handled"

deploy/docker/tests/cli/advanced/test_08_restart_with_scale.sh

@@ -1,56 +0,0 @@
-#!/bin/bash
-# Test: Restart server with different replica count
-# Expected: Server restarts with new replica count
-
-set -e
-
-echo "=== Test: Restart with Scale Change ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start with 2 replicas
-echo "Starting server with 2 replicas..."
-crwl server start --replicas 2 >/dev/null 2>&1
-sleep 8
-
-# Verify 2 replicas
-STATUS=$(crwl server status | grep "Replicas" || echo "")
-echo "Initial: $STATUS"
-
-# Restart with 4 replicas
-echo ""
-echo "Restarting with 4 replicas..."
-crwl server restart --replicas 4
-
-sleep 10
-
-# Verify 4 replicas
-STATUS=$(crwl server status)
-echo "$STATUS"
-
-if ! echo "$STATUS" | grep -q "4"; then
-    echo "❌ Status does not show 4 replicas after restart"
-    crwl server stop
-    exit 1
-fi
-
-# Verify health
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "❌ Health check failed after restart"
-    crwl server stop
-    exit 1
-fi
-
-# Cleanup
-echo "Cleaning up..."
-crwl server stop >/dev/null 2>&1
-
-echo ""
-echo "✅ Test passed: Server restarted with new replica count"

deploy/docker/tests/cli/basic/test_01_start_default.sh

@@ -1,52 +0,0 @@
-#!/bin/bash
-# Test: Start server with default settings
-# Expected: Server starts with 1 replica on port 11235
-
-set -e
-
-echo "=== Test: Start Server with Defaults ==="
-echo "Expected: 1 replica, port 11235, auto mode"
-echo ""
-
-# Activate virtual environment
-# Navigate to project root and activate venv
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup any existing server
-echo "Cleaning up any existing server..."
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start server with defaults
-echo "Starting server with default settings..."
-crwl server start
-
-# Wait for server to be ready
-echo "Waiting for server to be healthy..."
-sleep 5
-
-# Verify server is running
-echo "Checking server status..."
-STATUS=$(crwl server status | grep "Running" || echo "NOT_RUNNING")
-if [[ "$STATUS" == "NOT_RUNNING" ]]; then
-    echo "❌ Server failed to start"
-    crwl server stop
-    exit 1
-fi
-
-# Check health endpoint
-echo "Checking health endpoint..."
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "❌ Health check failed: $HEALTH"
-    crwl server stop
-    exit 1
-fi
-
-# Cleanup
-echo "Cleaning up..."
-crwl server stop
-
-echo ""
-echo "✅ Test passed: Server started with defaults and responded to health check"

deploy/docker/tests/cli/basic/test_02_status.sh

@@ -1,42 +0,0 @@
-#!/bin/bash
-# Test: Check server status command
-# Expected: Shows running status with correct details
-
-set -e
-
-echo "=== Test: Server Status Command ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Start server first
-echo "Starting server..."
-crwl server start >/dev/null 2>&1
-sleep 5
-
-# Check status
-echo "Checking server status..."
-STATUS_OUTPUT=$(crwl server status)
-echo "$STATUS_OUTPUT"
-echo ""
-
-# Verify output contains expected fields
-if ! echo "$STATUS_OUTPUT" | grep -q "Running"; then
-    echo "❌ Status does not show 'Running'"
-    crwl server stop
-    exit 1
-fi
-
-if ! echo "$STATUS_OUTPUT" | grep -q "11235"; then
-    echo "❌ Status does not show correct port"
-    crwl server stop
-    exit 1
-fi
-
-# Cleanup
-echo "Cleaning up..."
-crwl server stop >/dev/null 2>&1
-
-echo ""
-echo "✅ Test passed: Status command shows correct information"

deploy/docker/tests/cli/basic/test_03_stop.sh

@@ -1,45 +0,0 @@
-#!/bin/bash
-# Test: Stop server command
-# Expected: Server stops cleanly and port becomes available
-
-set -e
-
-echo "=== Test: Stop Server Command ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Start server first
-echo "Starting server..."
-crwl server start >/dev/null 2>&1
-sleep 5
-
-# Verify running
-echo "Verifying server is running..."
-if ! curl -s http://localhost:11235/health > /dev/null 2>&1; then
-    echo "❌ Server is not running before stop"
-    exit 1
-fi
-
-# Stop server
-echo "Stopping server..."
-crwl server stop
-
-# Verify stopped
-echo "Verifying server is stopped..."
-sleep 3
-if curl -s http://localhost:11235/health > /dev/null 2>&1; then
-    echo "❌ Server is still responding after stop"
-    exit 1
-fi
-
-# Check status shows not running
-STATUS=$(crwl server status | grep "No server" || echo "RUNNING")
-if [[ "$STATUS" == "RUNNING" ]]; then
-    echo "❌ Status still shows server as running"
-    exit 1
-fi
-
-echo ""
-echo "✅ Test passed: Server stopped cleanly"

deploy/docker/tests/cli/basic/test_04_start_custom_port.sh

@@ -1,46 +0,0 @@
-#!/bin/bash
-# Test: Start server with custom port
-# Expected: Server starts on port 8080 instead of default 11235
-
-set -e
-
-echo "=== Test: Start Server with Custom Port ==="
-echo "Expected: Server on port 8080"
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start on custom port
-echo "Starting server on port 8080..."
-crwl server start --port 8080
-
-sleep 5
-
-# Check health on custom port
-echo "Checking health on port 8080..."
-HEALTH=$(curl -s http://localhost:8080/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "❌ Health check failed on port 8080: $HEALTH"
-    crwl server stop
-    exit 1
-fi
-
-# Verify default port is NOT responding
-echo "Verifying port 11235 is not in use..."
-if curl -s http://localhost:11235/health > /dev/null 2>&1; then
-    echo "❌ Server is also running on default port 11235"
-    crwl server stop
-    exit 1
-fi
-
-# Cleanup
-echo "Cleaning up..."
-crwl server stop
-
-echo ""
-echo "✅ Test passed: Server started on custom port 8080"

deploy/docker/tests/cli/basic/test_05_start_replicas.sh

@@ -1,54 +0,0 @@
-#!/bin/bash
-# Test: Start server with multiple replicas
-# Expected: Server starts with 3 replicas in compose mode
-
-set -e
-
-echo "=== Test: Start Server with 3 Replicas ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start with 3 replicas
-echo "Starting server with 3 replicas..."
-crwl server start --replicas 3
-
-sleep 10
-
-# Check status shows 3 replicas
-echo "Checking status..."
-STATUS_OUTPUT=$(crwl server status)
-echo "$STATUS_OUTPUT"
-
-if ! echo "$STATUS_OUTPUT" | grep -q "3"; then
-    echo "❌ Status does not show 3 replicas"
-    crwl server stop
-    exit 1
-fi
-
-# Check health endpoint
-echo "Checking health endpoint..."
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "❌ Health check failed"
-    crwl server stop
-    exit 1
-fi
-
-# Check container discovery (should show 3 containers eventually)
-echo "Checking container discovery..."
-sleep 5  # Wait for heartbeats
-CONTAINERS=$(curl -s http://localhost:11235/monitor/containers | jq -r '.count' 2>/dev/null || echo "0")
-echo "Container count: $CONTAINERS"
-
-# Cleanup
-echo "Cleaning up..."
-crwl server stop
-
-echo ""
-echo "✅ Test passed: Server started with 3 replicas"

deploy/docker/tests/cli/basic/test_06_logs.sh

@@ -1,47 +0,0 @@
-#!/bin/bash
-# Test: View server logs
-# Expected: Logs are displayed without errors
-
-set -e
-
-echo "=== Test: Server Logs Command ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Start server
-echo "Starting server..."
-crwl server start >/dev/null 2>&1
-sleep 5
-
-# Make a request to generate some logs
-echo "Making request to generate logs..."
-curl -s http://localhost:11235/health > /dev/null
-
-# Check logs (tail)
-echo "Fetching logs (last 50 lines)..."
-LOGS=$(crwl server logs --tail 50 2>&1 || echo "ERROR")
-if [[ "$LOGS" == "ERROR" ]]; then
-    echo "❌ Failed to retrieve logs"
-    crwl server stop
-    exit 1
-fi
-
-echo "Log sample (first 10 lines):"
-echo "$LOGS" | head -n 10
-echo ""
-
-# Verify logs contain something (not empty)
-if [[ -z "$LOGS" ]]; then
-    echo "❌ Logs are empty"
-    crwl server stop
-    exit 1
-fi
-
-# Cleanup
-echo "Cleaning up..."
-crwl server stop >/dev/null 2>&1
-
-echo ""
-echo "✅ Test passed: Logs retrieved successfully"

deploy/docker/tests/cli/basic/test_07_restart.sh

@@ -1,55 +0,0 @@
-#!/bin/bash
-# Test: Restart server command
-# Expected: Server restarts with same configuration
-
-set -e
-
-echo "=== Test: Restart Server Command ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Start server with specific config
-echo "Starting server with 2 replicas..."
-crwl server start --replicas 2 >/dev/null 2>&1
-sleep 8
-
-# Get initial container ID
-echo "Getting initial state..."
-INITIAL_STATUS=$(crwl server status)
-echo "$INITIAL_STATUS"
-
-# Restart
-echo ""
-echo "Restarting server..."
-crwl server restart
-
-sleep 8
-
-# Check status after restart
-echo "Checking status after restart..."
-RESTART_STATUS=$(crwl server status)
-echo "$RESTART_STATUS"
-
-# Verify still has 2 replicas
-if ! echo "$RESTART_STATUS" | grep -q "2"; then
-    echo "❌ Replica count not preserved after restart"
-    crwl server stop
-    exit 1
-fi
-
-# Verify health
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "❌ Health check failed after restart"
-    crwl server stop
-    exit 1
-fi
-
-# Cleanup
-echo "Cleaning up..."
-crwl server stop >/dev/null 2>&1
-
-echo ""
-echo "✅ Test passed: Server restarted with preserved configuration"

deploy/docker/tests/cli/basic/test_08_cleanup.sh

@@ -1,46 +0,0 @@
-#!/bin/bash
-# Test: Force cleanup command
-# Expected: All resources removed even if state is corrupted
-
-set -e
-
-echo "=== Test: Force Cleanup Command ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Start server
-echo "Starting server..."
-crwl server start >/dev/null 2>&1
-sleep 5
-
-# Run cleanup (will prompt, so use force flag)
-echo "Running force cleanup..."
-crwl server cleanup --force
-
-sleep 3
-
-# Verify no containers running
-echo "Verifying cleanup..."
-CONTAINERS=$(docker ps --filter "name=crawl4ai" --format "{{.Names}}" || echo "")
-if [[ -n "$CONTAINERS" ]]; then
-    echo "❌ Crawl4AI containers still running: $CONTAINERS"
-    exit 1
-fi
-
-# Verify port is free
-if curl -s http://localhost:11235/health > /dev/null 2>&1; then
-    echo "❌ Server still responding after cleanup"
-    exit 1
-fi
-
-# Verify status shows not running
-STATUS=$(crwl server status | grep "No server" || echo "RUNNING")
-if [[ "$STATUS" == "RUNNING" ]]; then
-    echo "❌ Status still shows server running after cleanup"
-    exit 1
-fi
-
-echo ""
-echo "✅ Test passed: Force cleanup removed all resources"

deploy/docker/tests/cli/dashboard/run_dashboard_test.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-# Wrapper script to run dashboard UI test with proper environment
-
-set -e
-
-echo "=== Dashboard UI Test ==="
-echo ""
-
-# Activate virtual environment
-source venv/bin/activate
-
-# Make sure playwright is installed
-echo "Checking Playwright installation..."
-python -c "import playwright" 2>/dev/null || {
-    echo "Installing Playwright..."
-    pip install playwright
-    playwright install chromium
-}
-
-# Run the test
-echo ""
-echo "Running dashboard UI test..."
-python deploy/docker/tests/cli/dashboard/test_01_dashboard_ui.py
-
-echo ""
-echo "✅ Dashboard test complete"
-echo "Check deploy/docker/tests/cli/dashboard/screenshots/ for results"

deploy/docker/tests/cli/dashboard/test_01_dashboard_ui.py

@@ -1,225 +0,0 @@
-#!/usr/bin/env python3
-"""
-Dashboard UI Test with Playwright
-Tests the monitoring dashboard UI functionality
-"""
-import asyncio
-import subprocess
-import time
-import os
-from pathlib import Path
-from playwright.async_api import async_playwright
-
-BASE_URL = "http://localhost:11235"
-SCREENSHOT_DIR = Path(__file__).parent / "screenshots"
-
-async def start_server():
-    """Start server with 3 replicas"""
-    print("Starting server with 3 replicas...")
-    subprocess.run(["crwl", "server", "stop"],
-                   stdout=subprocess.DEVNULL,
-                   stderr=subprocess.DEVNULL)
-    time.sleep(2)
-
-    result = subprocess.run(
-        ["crwl", "server", "start", "--replicas", "3"],
-        capture_output=True,
-        text=True
-    )
-
-    if result.returncode != 0:
-        raise Exception(f"Failed to start server: {result.stderr}")
-
-    print("Waiting for server to be ready...")
-    time.sleep(12)
-
-async def run_demo_script():
-    """Run the demo script in background to generate activity"""
-    print("Starting demo script to generate dashboard activity...")
-    demo_path = Path(__file__).parent.parent.parent / "monitor" / "demo_monitor_dashboard.py"
-
-    process = subprocess.Popen(
-        ["python", str(demo_path)],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE
-    )
-
-    # Let it run for a bit to generate some data
-    print("Waiting for demo to generate data...")
-    time.sleep(10)
-
-    return process
-
-async def test_dashboard_ui():
-    """Test dashboard UI with Playwright"""
-
-    # Create screenshot directory
-    SCREENSHOT_DIR.mkdir(exist_ok=True)
-    print(f"Screenshots will be saved to: {SCREENSHOT_DIR}")
-
-    async with async_playwright() as p:
-        # Launch browser
-        print("\nLaunching browser...")
-        browser = await p.chromium.launch(headless=True)
-        context = await browser.new_context(viewport={'width': 1920, 'height': 1080})
-        page = await context.new_page()
-
-        try:
-            # Navigate to dashboard
-            print(f"Navigating to {BASE_URL}/dashboard")
-            await page.goto(f"{BASE_URL}/dashboard", wait_until="networkidle")
-            await asyncio.sleep(3)
-
-            # Take full dashboard screenshot
-            print("Taking full dashboard screenshot...")
-            await page.screenshot(path=SCREENSHOT_DIR / "01_full_dashboard.png", full_page=True)
-            print(f"  ✅ Saved: 01_full_dashboard.png")
-
-            # Verify page title
-            title = await page.title()
-            print(f"\nPage title: {title}")
-            if "Monitor" not in title and "Dashboard" not in title:
-                print("  ⚠️  Warning: Title doesn't contain 'Monitor' or 'Dashboard'")
-
-            # Check for infrastructure card (container filters)
-            print("\nChecking Infrastructure card...")
-            infrastructure = await page.query_selector('.card h3:has-text("Infrastructure")')
-            if infrastructure:
-                print("  ✅ Infrastructure card found")
-                await page.screenshot(path=SCREENSHOT_DIR / "02_infrastructure_card.png")
-                print(f"  ✅ Saved: 02_infrastructure_card.png")
-            else:
-                print("  ❌ Infrastructure card not found")
-
-            # Check for container filter buttons (All, C-1, C-2, C-3)
-            print("\nChecking container filter buttons...")
-            all_button = await page.query_selector('.filter-btn[data-container="all"]')
-            if all_button:
-                print("  ✅ 'All' filter button found")
-                # Take screenshot of filter area
-                await all_button.screenshot(path=SCREENSHOT_DIR / "03_filter_buttons.png")
-                print(f"  ✅ Saved: 03_filter_buttons.png")
-
-                # Test clicking filter button
-                await all_button.click()
-                await asyncio.sleep(1)
-                print("  ✅ Clicked 'All' filter button")
-            else:
-                print("  ⚠️  'All' filter button not found (may appear after containers register)")
-
-            # Check for WebSocket connection indicator
-            print("\nChecking WebSocket connection...")
-            ws_indicator = await page.query_selector('.ws-status, .connection-status, [class*="websocket"]')
-            if ws_indicator:
-                print("  ✅ WebSocket indicator found")
-            else:
-                print("  ⚠️  WebSocket indicator not found in DOM")
-
-            # Check for main dashboard sections
-            print("\nChecking dashboard sections...")
-            sections = [
-                ("Active Requests", ".active-requests, [class*='active']"),
-                ("Completed Requests", ".completed-requests, [class*='completed']"),
-                ("Browsers", ".browsers, [class*='browser']"),
-                ("Timeline", ".timeline, [class*='timeline']"),
-            ]
-
-            for section_name, selector in sections:
-                element = await page.query_selector(selector)
-                if element:
-                    print(f"  ✅ {section_name} section found")
-                else:
-                    print(f"  ⚠️  {section_name} section not found with selector: {selector}")
-
-            # Scroll to different sections and take screenshots
-            print("\nTaking section screenshots...")
-
-            # Requests section
-            requests = await page.query_selector('.card h3:has-text("Requests")')
-            if requests:
-                await requests.scroll_into_view_if_needed()
-                await asyncio.sleep(1)
-                await page.screenshot(path=SCREENSHOT_DIR / "04_requests_section.png")
-                print(f"  ✅ Saved: 04_requests_section.png")
-
-            # Browsers section
-            browsers = await page.query_selector('.card h3:has-text("Browsers")')
-            if browsers:
-                await browsers.scroll_into_view_if_needed()
-                await asyncio.sleep(1)
-                await page.screenshot(path=SCREENSHOT_DIR / "05_browsers_section.png")
-                print(f"  ✅ Saved: 05_browsers_section.png")
-
-            # Timeline section
-            timeline = await page.query_selector('.card h3:has-text("Timeline")')
-            if timeline:
-                await timeline.scroll_into_view_if_needed()
-                await asyncio.sleep(1)
-                await page.screenshot(path=SCREENSHOT_DIR / "06_timeline_section.png")
-                print(f"  ✅ Saved: 06_timeline_section.png")
-
-            # Check for tabs (if they exist)
-            print("\nChecking for tabs...")
-            tabs = await page.query_selector_all('.tab, [role="tab"]')
-            if tabs:
-                print(f"  ✅ Found {len(tabs)} tabs")
-                for i, tab in enumerate(tabs[:5]):  # Check first 5 tabs
-                    tab_text = await tab.inner_text()
-                    print(f"    - Tab {i+1}: {tab_text}")
-            else:
-                print("  ℹ️  No tab elements found")
-
-            # Wait for any animations to complete
-            await asyncio.sleep(2)
-
-            # Take final screenshot
-            print("\nTaking final screenshot...")
-            await page.screenshot(path=SCREENSHOT_DIR / "07_final_state.png", full_page=True)
-            print(f"  ✅ Saved: 07_final_state.png")
-
-            print("\n" + "="*60)
-            print("Dashboard UI Test Complete!")
-            print(f"Screenshots saved to: {SCREENSHOT_DIR}")
-            print("="*60)
-
-        finally:
-            await browser.close()
-
-async def cleanup():
-    """Stop server and cleanup"""
-    print("\nCleaning up...")
-    subprocess.run(["crwl", "server", "stop"],
-                   stdout=subprocess.DEVNULL,
-                   stderr=subprocess.DEVNULL)
-    print("✅ Cleanup complete")
-
-async def main():
-    """Main test execution"""
-    demo_process = None
-
-    try:
-        # Start server
-        await start_server()
-
-        # Run demo script to generate activity
-        demo_process = await run_demo_script()
-
-        # Run dashboard UI test
-        await test_dashboard_ui()
-
-        print("\n✅ All dashboard UI tests passed!")
-
-    except Exception as e:
-        print(f"\n❌ Test failed: {e}")
-        raise
-    finally:
-        # Stop demo script
-        if demo_process:
-            demo_process.terminate()
-            demo_process.wait(timeout=5)
-
-        # Cleanup server
-        await cleanup()
-
-if __name__ == "__main__":
-    asyncio.run(main())

deploy/docker/tests/cli/edge/test_01_already_running.sh

@@ -1,48 +0,0 @@
-#!/bin/bash
-# Test: Try starting server when already running
-# Expected: Error message indicating server is already running
-
-set -e
-
-echo "=== Test: Start When Already Running ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start server
-echo "Starting server..."
-crwl server start >/dev/null 2>&1
-sleep 5
-
-# Try to start again
-echo ""
-echo "Attempting to start server again (should fail)..."
-OUTPUT=$(crwl server start 2>&1 || true)
-echo "$OUTPUT"
-
-# Verify error message
-if echo "$OUTPUT" | grep -iq "already running"; then
-    echo ""
-    echo "✅ Test passed: Proper error for already running server"
-else
-    echo ""
-    echo "❌ Test failed: Expected 'already running' error message"
-    crwl server stop
-    exit 1
-fi
-
-# Verify original server still running
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "❌ Original server is not running"
-    crwl server stop
-    exit 1
-fi
-
-# Cleanup
-crwl server stop >/dev/null 2>&1

deploy/docker/tests/cli/edge/test_02_not_running.sh

@@ -1,50 +0,0 @@
-#!/bin/bash
-# Test: Operations when server is not running
-# Expected: Appropriate error messages
-
-set -e
-
-echo "=== Test: Operations When Not Running ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Make sure nothing is running
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Try status when not running
-echo "Checking status when not running..."
-OUTPUT=$(crwl server status 2>&1 || true)
-echo "$OUTPUT"
-echo ""
-
-if ! echo "$OUTPUT" | grep -iq "no server"; then
-    echo "❌ Status should indicate no server running"
-    exit 1
-fi
-
-# Try stop when not running
-echo "Trying to stop when not running..."
-OUTPUT=$(crwl server stop 2>&1 || true)
-echo "$OUTPUT"
-echo ""
-
-if ! echo "$OUTPUT" | grep -iq "no server\|not running"; then
-    echo "❌ Stop should indicate no server running"
-    exit 1
-fi
-
-# Try scale when not running
-echo "Trying to scale when not running..."
-OUTPUT=$(crwl server scale 3 2>&1 || true)
-echo "$OUTPUT"
-echo ""
-
-if ! echo "$OUTPUT" | grep -iq "no server\|not running"; then
-    echo "❌ Scale should indicate no server running"
-    exit 1
-fi
-
-echo "✅ Test passed: Appropriate errors for operations when not running"

deploy/docker/tests/cli/edge/test_03_scale_single_mode.sh

@@ -1,47 +0,0 @@
-#!/bin/bash
-# Test: Try to scale single container mode
-# Expected: Error indicating single mode cannot be scaled
-
-set -e
-
-echo "=== Test: Scale Single Container Mode ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start in single mode
-echo "Starting in single mode..."
-crwl server start --mode single >/dev/null 2>&1
-sleep 5
-
-# Try to scale
-echo ""
-echo "Attempting to scale single mode (should fail)..."
-OUTPUT=$(crwl server scale 3 2>&1 || true)
-echo "$OUTPUT"
-echo ""
-
-# Verify error message
-if echo "$OUTPUT" | grep -iq "single"; then
-    echo "✅ Test passed: Proper error for scaling single mode"
-else
-    echo "❌ Test failed: Expected error about single mode"
-    crwl server stop
-    exit 1
-fi
-
-# Verify server still running
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "❌ Server is not running after failed scale"
-    crwl server stop
-    exit 1
-fi
-
-# Cleanup
-crwl server stop >/dev/null 2>&1

deploy/docker/tests/cli/edge/test_04_invalid_port.sh

@@ -1,36 +0,0 @@
-#!/bin/bash
-# Test: Invalid port numbers
-# Expected: Validation errors for invalid ports
-
-set -e
-
-echo "=== Test: Invalid Port Numbers ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Test invalid ports
-INVALID_PORTS=(0 -1 99999 65536)
-
-for PORT in "${INVALID_PORTS[@]}"; do
-    echo "Testing invalid port: $PORT"
-    OUTPUT=$(crwl server start --port $PORT 2>&1 || true)
-
-    if echo "$OUTPUT" | grep -iq "error\|invalid\|usage"; then
-        echo "  ✅ Rejected port $PORT"
-    else
-        echo "  ⚠️  Port $PORT may have been accepted (output: $OUTPUT)"
-    fi
-
-    # Make sure no server started
-    crwl server stop 2>/dev/null || true
-    sleep 1
-    echo ""
-done
-
-echo "✅ Test passed: Invalid ports handled appropriately"

deploy/docker/tests/cli/edge/test_05_invalid_replicas.sh

@@ -1,57 +0,0 @@
-#!/bin/bash
-# Test: Invalid replica counts
-# Expected: Validation errors for invalid replicas
-
-set -e
-
-echo "=== Test: Invalid Replica Counts ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Test invalid replica counts
-INVALID_REPLICAS=(0 -1 101)
-
-for REPLICAS in "${INVALID_REPLICAS[@]}"; do
-    echo "Testing invalid replica count: $REPLICAS"
-    OUTPUT=$(crwl server start --replicas $REPLICAS 2>&1 || true)
-
-    if echo "$OUTPUT" | grep -iq "error\|invalid\|usage"; then
-        echo "  ✅ Rejected replica count $REPLICAS"
-    else
-        echo "  ⚠️  Replica count $REPLICAS may have been accepted"
-    fi
-
-    # Make sure no server started
-    crwl server stop 2>/dev/null || true
-    sleep 1
-    echo ""
-done
-
-# Test scaling to invalid counts
-echo "Testing scale to invalid counts..."
-crwl server start --replicas 2 >/dev/null 2>&1
-sleep 5
-
-INVALID_SCALE=(0 -1)
-for SCALE in "${INVALID_SCALE[@]}"; do
-    echo "Testing scale to: $SCALE"
-    OUTPUT=$(crwl server scale $SCALE 2>&1 || true)
-
-    if echo "$OUTPUT" | grep -iq "error\|invalid\|must be at least 1"; then
-        echo "  ✅ Rejected scale to $SCALE"
-    else
-        echo "  ⚠️  Scale to $SCALE may have been accepted"
-    fi
-    echo ""
-done
-
-# Cleanup
-crwl server stop >/dev/null 2>&1
-
-echo "✅ Test passed: Invalid replica counts handled appropriately"

deploy/docker/tests/cli/edge/test_06_missing_env_file.sh

@@ -1,40 +0,0 @@
-#!/bin/bash
-# Test: Non-existent environment file
-# Expected: Error indicating file not found
-
-set -e
-
-echo "=== Test: Missing Environment File ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Try with non-existent file
-FAKE_FILE="/tmp/nonexistent_$(date +%s).env"
-echo "Attempting to start with non-existent env file: $FAKE_FILE"
-OUTPUT=$(crwl server start --env-file "$FAKE_FILE" 2>&1 || true)
-echo "$OUTPUT"
-echo ""
-
-# Verify error
-if echo "$OUTPUT" | grep -iq "error\|does not exist\|not found\|no such file"; then
-    echo "✅ Test passed: Proper error for missing env file"
-else
-    echo "❌ Test failed: Expected error about missing file"
-    crwl server stop
-    exit 1
-fi
-
-# Make sure no server started
-if curl -s http://localhost:11235/health > /dev/null 2>&1; then
-    echo "❌ Server should not have started"
-    crwl server stop
-    exit 1
-fi
-
-echo "✅ Server correctly refused to start with missing env file"

deploy/docker/tests/cli/edge/test_07_port_in_use.sh

@@ -1,50 +0,0 @@
-#!/bin/bash
-# Test: Port already in use
-# Expected: Error indicating port is occupied
-
-set -e
-
-echo "=== Test: Port Already In Use ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start a simple HTTP server on port 11235 to occupy it
-echo "Starting dummy server on port 11235..."
-python -m http.server 11235 >/dev/null 2>&1 &
-DUMMY_PID=$!
-sleep 2
-
-# Try to start crawl4ai on same port
-echo "Attempting to start Crawl4AI on occupied port..."
-OUTPUT=$(crwl server start 2>&1 || true)
-echo "$OUTPUT"
-echo ""
-
-# Kill dummy server
-kill $DUMMY_PID 2>/dev/null || true
-sleep 1
-
-# Verify error message
-if echo "$OUTPUT" | grep -iq "port.*in use\|already in use\|address already in use"; then
-    echo "✅ Test passed: Proper error for port in use"
-else
-    echo "⚠️  Expected 'port in use' error (output may vary)"
-fi
-
-# Make sure Crawl4AI didn't start
-if curl -s http://localhost:11235/health > /dev/null 2>&1; then
-    HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "unknown")
-    if [[ "$HEALTH" == "ok" ]]; then
-        echo "❌ Crawl4AI started despite port being occupied"
-        crwl server stop
-        exit 1
-    fi
-fi
-
-echo "✅ Crawl4AI correctly refused to start on occupied port"

deploy/docker/tests/cli/edge/test_08_state_corruption.sh

@@ -1,79 +0,0 @@
-#!/bin/bash
-# Test: Corrupted state file
-# Expected: Cleanup recovers from corrupted state
-
-set -e
-
-echo "=== Test: State File Corruption ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start server to create state
-echo "Starting server to create state..."
-crwl server start >/dev/null 2>&1
-sleep 5
-
-# Get state file path
-STATE_FILE="$HOME/.crawl4ai/server/state.json"
-echo "State file: $STATE_FILE"
-
-# Verify state file exists
-if [[ ! -f "$STATE_FILE" ]]; then
-    echo "❌ State file not created"
-    crwl server stop
-    exit 1
-fi
-
-echo "Original state:"
-cat "$STATE_FILE" | jq '.' || cat "$STATE_FILE"
-echo ""
-
-# Stop server
-crwl server stop >/dev/null 2>&1
-sleep 2
-
-# Corrupt state file
-echo "Corrupting state file..."
-echo "{ invalid json }" > "$STATE_FILE"
-cat "$STATE_FILE"
-echo ""
-
-# Try to start server (should handle corrupted state)
-echo "Attempting to start with corrupted state..."
-OUTPUT=$(crwl server start 2>&1 || true)
-echo "$OUTPUT"
-echo ""
-
-# Check if server started or gave clear error
-if curl -s http://localhost:11235/health > /dev/null 2>&1; then
-    echo "✅ Server started despite corrupted state"
-    crwl server stop
-elif echo "$OUTPUT" | grep -iq "already running"; then
-    # State thinks server is running, use cleanup
-    echo "State thinks server is running, using cleanup..."
-    crwl server cleanup --force >/dev/null 2>&1
-    sleep 2
-
-    # Try starting again
-    crwl server start >/dev/null 2>&1
-    sleep 5
-
-    if curl -s http://localhost:11235/health > /dev/null 2>&1; then
-        echo "✅ Cleanup recovered from corrupted state"
-        crwl server stop
-    else
-        echo "❌ Failed to recover from corrupted state"
-        exit 1
-    fi
-else
-    echo "✅ Handled corrupted state appropriately"
-fi
-
-echo ""
-echo "✅ Test passed: System handles state corruption"

deploy/docker/tests/cli/edge/test_09_network_conflict.sh

@@ -1,47 +0,0 @@
-#!/bin/bash
-# Test: Docker network name collision
-# Expected: Handles existing network gracefully
-
-set -e
-
-echo "=== Test: Network Name Conflict ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Create a network with similar name
-NETWORK_NAME="crawl4ai_test_net"
-echo "Creating test network: $NETWORK_NAME..."
-docker network create "$NETWORK_NAME" 2>/dev/null || echo "Network may already exist"
-
-# Start server (should either use existing network or create its own)
-echo ""
-echo "Starting server..."
-crwl server start >/dev/null 2>&1
-sleep 5
-
-# Verify server started successfully
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "❌ Server failed to start"
-    docker network rm "$NETWORK_NAME" 2>/dev/null || true
-    crwl server stop
-    exit 1
-fi
-
-echo "✅ Server started successfully despite network conflict"
-
-# Cleanup
-crwl server stop >/dev/null 2>&1
-sleep 2
-
-# Remove test network
-docker network rm "$NETWORK_NAME" 2>/dev/null || echo "Network already removed"
-
-echo ""
-echo "✅ Test passed: Handled network conflict gracefully"

deploy/docker/tests/cli/edge/test_10_rapid_operations.sh

@@ -1,72 +0,0 @@
-#!/bin/bash
-# Test: Rapid start/stop/restart operations
-# Expected: System handles rapid operations without corruption
-
-set -e
-
-echo "=== Test: Rapid Operations ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Test 1: Rapid start/stop
-echo "Test 1: Rapid start/stop cycles..."
-for i in {1..3}; do
-    echo "  Cycle $i/3..."
-    crwl server start >/dev/null 2>&1
-    sleep 3
-    crwl server stop >/dev/null 2>&1
-    sleep 2
-done
-echo "  ✅ Completed rapid start/stop cycles"
-
-# Test 2: Restart immediately after start
-echo ""
-echo "Test 2: Restart immediately after start..."
-crwl server start >/dev/null 2>&1
-sleep 3
-crwl server restart >/dev/null 2>&1
-sleep 5
-
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "  ❌ Health check failed after rapid restart"
-    crwl server stop
-    exit 1
-fi
-echo "  ✅ Rapid restart successful"
-
-# Test 3: Multiple status checks
-echo ""
-echo "Test 3: Multiple rapid status checks..."
-for i in {1..5}; do
-    crwl server status >/dev/null 2>&1 || echo "  ⚠️  Status check $i failed"
-done
-echo "  ✅ Multiple status checks completed"
-
-# Test 4: Stop and immediate start
-echo ""
-echo "Test 4: Stop and immediate start..."
-crwl server stop >/dev/null 2>&1
-sleep 2
-crwl server start >/dev/null 2>&1
-sleep 5
-
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "  ❌ Health check failed after stop/start"
-    crwl server stop
-    exit 1
-fi
-echo "  ✅ Stop/immediate start successful"
-
-# Cleanup
-crwl server stop >/dev/null 2>&1
-
-echo ""
-echo "✅ Test passed: System handles rapid operations correctly"

deploy/docker/tests/cli/plan.md

@@ -1,119 +0,0 @@
-E2E CLI Test Suite Plan                                                                                     │ │
-│ │                                                                                                             │ │
-│ │ Test Structure                                                                                              │ │
-│ │                                                                                                             │ │
-│ │ Create deploy/docker/tests/cli/ folder with individual test scripts organized by category.                  │ │
-│ │                                                                                                             │ │
-│ │ Test Categories                                                                                             │ │
-│ │                                                                                                             │ │
-│ │ 1. Basic Tests (deploy/docker/tests/cli/basic/)                                                             │ │
-│ │                                                                                                             │ │
-│ │ - test_01_start_default.sh - Start server with defaults (1 replica, port 11235)                             │ │
-│ │ - test_02_status.sh - Check server status                                                                   │ │
-│ │ - test_03_stop.sh - Stop server cleanly                                                                     │ │
-│ │ - test_04_start_custom_port.sh - Start with custom port (8080)                                              │ │
-│ │ - test_05_start_replicas.sh - Start with 3 replicas                                                         │ │
-│ │ - test_06_logs.sh - View logs (tail and follow)                                                             │ │
-│ │ - test_07_restart.sh - Restart server preserving config                                                     │ │
-│ │ - test_08_cleanup.sh - Force cleanup all resources                                                          │ │
-│ │                                                                                                             │ │
-│ │ 2. Advanced Tests (deploy/docker/tests/cli/advanced/)                                                       │ │
-│ │                                                                                                             │ │
-│ │ - test_01_scale_up.sh - Scale from 3 to 5 replicas                                                          │ │
-│ │ - test_02_scale_down.sh - Scale from 5 to 2 replicas                                                        │ │
-│ │ - test_03_mode_single.sh - Start in single mode explicitly                                                  │ │
-│ │ - test_04_mode_compose.sh - Start in compose mode with 3 replicas                                           │ │
-│ │ - test_05_custom_image.sh - Start with custom image tag                                                     │ │
-│ │ - test_06_env_file.sh - Start with custom env file                                                          │ │
-│ │ - test_07_stop_remove_volumes.sh - Stop and remove volumes                                                  │ │
-│ │ - test_08_restart_with_scale.sh - Restart and change replica count                                          │ │
-│ │                                                                                                             │ │
-│ │ 3. Resource Tests (deploy/docker/tests/cli/resource/)                                                       │ │
-│ │                                                                                                             │ │
-│ │ - test_01_memory_monitoring.sh - Monitor memory during crawls                                               │ │
-│ │ - test_02_cpu_stress.sh - CPU usage under concurrent load                                                   │ │
-│ │ - test_03_max_replicas.sh - Start with 10 replicas and stress test                                          │ │
-│ │ - test_04_cleanup_verification.sh - Verify all resources cleaned up                                         │ │
-│ │ - test_05_long_running.sh - Stability test (30 min runtime)                                                 │ │
-│ │                                                                                                             │ │
-│ │ 4. Dashboard UI Tests (deploy/docker/tests/cli/dashboard/)                                                  │ │
-│ │                                                                                                             │ │
-│ │ - test_01_dashboard_ui.py - Playwright test with screenshots                                                │ │
-│ │   - Start server with 3 replicas                                                                            │ │
-│ │   - Run demo_monitor_dashboard.py script                                                                    │ │
-│ │   - Use Playwright to:                                                                                      │ │
-│ │       - Take screenshot of main dashboard                                                                   │ │
-│ │     - Verify container filter buttons (All, C-1, C-2, C-3)                                                  │ │
-│ │     - Test WebSocket connection indicator                                                                   │ │
-│ │     - Verify timeline charts render                                                                         │ │
-│ │     - Test filtering functionality                                                                          │ │
-│ │     - Check all tabs (Requests, Browsers, Janitor, Errors, Stats)                                           │ │
-│ │                                                                                                             │ │
-│ │ 5. Edge Cases (deploy/docker/tests/cli/edge/)                                                               │ │
-│ │                                                                                                             │ │
-│ │ - test_01_already_running.sh - Try starting when already running                                            │ │
-│ │ - test_02_not_running.sh - Try stop/status when not running                                                 │ │
-│ │ - test_03_scale_single_mode.sh - Try scaling single container mode                                          │ │
-│ │ - test_04_invalid_port.sh - Invalid port numbers (0, -1, 99999)                                             │ │
-│ │ - test_05_invalid_replicas.sh - Invalid replica counts (0, -1, 101)                                         │ │
-│ │ - test_06_missing_env_file.sh - Non-existent env file                                                       │ │
-│ │ - test_07_port_in_use.sh - Port already occupied                                                            │ │
-│ │ - test_08_state_corruption.sh - Manually corrupt state file                                                 │ │
-│ │ - test_09_network_conflict.sh - Docker network name collision                                               │ │
-│ │ - test_10_rapid_operations.sh - Start/stop/restart in quick succession                                      │ │
-│ │                                                                                                             │ │
-│ │ Test Execution Plan                                                                                         │ │
-│ │                                                                                                             │ │
-│ │ Process:                                                                                                    │ │
-│ │                                                                                                             │ │
-│ │ 1. Create test file                                                                                         │ │
-│ │ 2. Run test                                                                                                 │ │
-│ │ 3. Verify results                                                                                           │ │
-│ │ 4. If fails → fix issue → re-test                                                                           │ │
-│ │ 5. Move to next test                                                                                        │ │
-│ │ 6. Clean up after each test to ensure clean state                                                           │ │
-│ │                                                                                                             │ │
-│ │ Common Test Structure:                                                                                      │ │
-│ │                                                                                                             │ │
-│ │ #!/bin/bash                                                                                                 │ │
-│ │ # Test: [Description]                                                                                       │ │
-│ │ # Expected: [What should happen]                                                                            │ │
-│ │                                                                                                             │ │
-│ │ source venv/bin/activate                                                                                    │ │
-│ │ set -e  # Exit on error                                                                                     │ │
-│ │                                                                                                             │ │
-│ │ echo "=== Test: [Name] ==="                                                                                 │ │
-│ │                                                                                                             │ │
-│ │ # Setup                                                                                                     │ │
-│ │ # ... test commands ...                                                                                     │ │
-│ │                                                                                                             │ │
-│ │ # Verification                                                                                              │ │
-│ │ # ... assertions ...                                                                                        │ │
-│ │                                                                                                             │ │
-│ │ # Cleanup                                                                                                   │ │
-│ │ crwl server stop || true                                                                                    │ │
-│ │                                                                                                             │ │
-│ │ echo "✓ Test passed"                                                                                        │ │
-│ │                                                                                                             │ │
-│ │ Dashboard Test Structure (Python):                                                                          │ │
-│ │                                                                                                             │ │
-│ │ # Activate venv first in calling script                                                                     │ │
-│ │ import asyncio                                                                                              │ │
-│ │ from playwright.async_api import async_playwright                                                           │ │
-│ │                                                                                                             │ │
-│ │ async def test_dashboard():                                                                                 │ │
-│ │     # Start server with 3 replicas                                                                          │ │
-│ │     # Run demo script in background                                                                         │ │
-│ │     # Launch Playwright                                                                                     │ │
-│ │     # Take screenshots                                                                                      │ │
-│ │     # Verify elements                                                                                       │ │
-│ │     # Cleanup                                                                                               │ │
-│ │                                                                                                             │ │
-│ │ Success Criteria:                                                                                           │ │
-│ │                                                                                                             │ │
-│ │ - All basic operations work correctly                                                                       │ │
-│ │ - Scaling operations function properly                                                                      │ │
-│ │ - Resource limits are respected                                                                             │ │
-│ │ - Dashboard UI is functional and responsive                                                                 │ │
-│ │ - Edge cases handled gracefully with proper error messages                                                  │ │
-│ │ - Clean resource cleanup verified

deploy/docker/tests/cli/resource/test_01_memory_monitoring.sh

@@ -1,63 +0,0 @@
-#!/bin/bash
-# Test: Monitor memory usage during crawl operations
-# Expected: Memory stats are accessible and reasonable
-
-set -e
-
-echo "=== Test: Memory Monitoring ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start server
-echo "Starting server..."
-crwl server start >/dev/null 2>&1
-sleep 5
-
-# Get baseline memory
-echo "Checking baseline memory..."
-BASELINE=$(curl -s http://localhost:11235/monitor/health | jq -r '.container.memory_percent' 2>/dev/null || echo "0")
-echo "Baseline memory: ${BASELINE}%"
-
-# Make several crawl requests
-echo ""
-echo "Making crawl requests to increase memory usage..."
-for i in {1..5}; do
-    echo "  Request $i/5..."
-    curl -s -X POST http://localhost:11235/crawl \
-      -H "Content-Type: application/json" \
-      -d "{\"urls\": [\"https://httpbin.org/html?req=$i\"], \"crawler_config\": {}}" > /dev/null || true
-    sleep 1
-done
-
-# Check memory after requests
-echo ""
-echo "Checking memory after requests..."
-AFTER=$(curl -s http://localhost:11235/monitor/health | jq -r '.container.memory_percent' 2>/dev/null || echo "0")
-echo "Memory after requests: ${AFTER}%"
-
-# Get browser pool stats
-echo ""
-echo "Browser pool memory usage..."
-POOL_MEM=$(curl -s http://localhost:11235/monitor/browsers | jq -r '.summary.total_memory_mb' 2>/dev/null || echo "0")
-echo "Browser pool: ${POOL_MEM} MB"
-
-# Verify memory is within reasonable bounds (<80%)
-MEMORY_OK=$(echo "$AFTER < 80" | bc -l 2>/dev/null || echo "1")
-if [[ "$MEMORY_OK" != "1" ]]; then
-    echo "⚠️  Warning: Memory usage is high: ${AFTER}%"
-fi
-
-# Cleanup
-echo ""
-echo "Cleaning up..."
-crwl server stop >/dev/null 2>&1
-
-echo ""
-echo "✅ Test passed: Memory monitoring functional"
-echo "   Baseline: ${BASELINE}%, After: ${AFTER}%, Pool: ${POOL_MEM} MB"

deploy/docker/tests/cli/resource/test_02_cpu_stress.sh

@@ -1,61 +0,0 @@
-#!/bin/bash
-# Test: CPU usage under concurrent load
-# Expected: Server handles concurrent requests without errors
-
-set -e
-
-echo "=== Test: CPU Stress Test ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start server with 3 replicas for better load distribution
-echo "Starting server with 3 replicas..."
-crwl server start --replicas 3 >/dev/null 2>&1
-sleep 12
-
-# Get baseline CPU
-echo "Checking baseline container stats..."
-docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}" \
-  --filter "name=crawl4ai" 2>/dev/null || echo "Unable to get container stats"
-
-# Send concurrent requests
-echo ""
-echo "Sending 10 concurrent requests..."
-for i in {1..10}; do
-    curl -s -X POST http://localhost:11235/crawl \
-      -H "Content-Type: application/json" \
-      -d "{\"urls\": [\"https://httpbin.org/html?req=$i\"], \"crawler_config\": {}}" > /dev/null &
-done
-
-# Wait for all requests to complete
-echo "Waiting for requests to complete..."
-wait
-
-# Check stats after load
-echo ""
-echo "Container stats after load:"
-docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}" \
-  --filter "name=crawl4ai" 2>/dev/null || echo "Unable to get container stats"
-
-# Verify health
-echo ""
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "❌ Health check failed after CPU stress"
-    crwl server stop
-    exit 1
-fi
-
-# Cleanup
-echo ""
-echo "Cleaning up..."
-crwl server stop >/dev/null 2>&1
-
-echo ""
-echo "✅ Test passed: Server handled concurrent load successfully"

deploy/docker/tests/cli/resource/test_03_max_replicas.sh

@@ -1,72 +0,0 @@
-#!/bin/bash
-# Test: Start with maximum replicas and stress test
-# Expected: Server handles max replicas (10) and distributes load
-
-set -e
-
-echo "=== Test: Maximum Replicas Stress Test ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start with 10 replicas (max recommended)
-echo "Starting server with 10 replicas..."
-echo "This may take some time..."
-crwl server start --replicas 10 >/dev/null 2>&1
-sleep 20
-
-# Verify status
-echo "Checking status..."
-STATUS=$(crwl server status)
-if ! echo "$STATUS" | grep -q "10"; then
-    echo "❌ Failed to start 10 replicas"
-    crwl server stop
-    exit 1
-fi
-
-# Wait for container discovery
-echo ""
-echo "Waiting for container discovery..."
-sleep 10
-
-# Check containers
-CONTAINER_COUNT=$(curl -s http://localhost:11235/monitor/containers | jq -r '.count' 2>/dev/null || echo "0")
-echo "Discovered containers: $CONTAINER_COUNT"
-
-# Send burst of requests
-echo ""
-echo "Sending burst of 20 requests..."
-for i in {1..20}; do
-    curl -s -X POST http://localhost:11235/crawl \
-      -H "Content-Type: application/json" \
-      -d "{\"urls\": [\"https://httpbin.org/html?req=$i\"], \"crawler_config\": {}}" > /dev/null &
-done
-
-wait
-
-# Check health after stress
-echo ""
-HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-if [[ "$HEALTH" != "ok" ]]; then
-    echo "❌ Health check failed after max replica stress"
-    crwl server stop
-    exit 1
-fi
-
-# Check endpoint stats
-echo ""
-echo "Endpoint statistics:"
-curl -s http://localhost:11235/monitor/endpoints/stats | jq '.' 2>/dev/null || echo "No stats available"
-
-# Cleanup
-echo ""
-echo "Cleaning up..."
-crwl server stop >/dev/null 2>&1
-
-echo ""
-echo "✅ Test passed: Successfully stress tested with 10 replicas"

deploy/docker/tests/cli/resource/test_04_cleanup_verification.sh

@@ -1,63 +0,0 @@
-#!/bin/bash
-# Test: Verify complete resource cleanup
-# Expected: All Docker resources are properly removed
-
-set -e
-
-echo "=== Test: Resource Cleanup Verification ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Start server to create resources
-echo "Starting server with 3 replicas..."
-crwl server start --replicas 3 >/dev/null 2>&1
-sleep 10
-
-# List resources before cleanup
-echo ""
-echo "Resources before cleanup:"
-echo "Containers:"
-docker ps --filter "name=crawl4ai" --format "  - {{.Names}}" 2>/dev/null || echo "  None"
-docker ps --filter "name=nginx" --format "  - {{.Names}}" 2>/dev/null || echo "  None"
-docker ps --filter "name=redis" --format "  - {{.Names}}" 2>/dev/null || echo "  None"
-
-echo ""
-echo "Networks:"
-docker network ls --filter "name=crawl4ai" --format "  - {{.Name}}" 2>/dev/null || echo "  None"
-
-# Cleanup
-echo ""
-echo "Performing cleanup..."
-crwl server cleanup --force >/dev/null 2>&1
-sleep 5
-
-# Verify cleanup
-echo ""
-echo "Verifying cleanup..."
-
-CONTAINERS=$(docker ps -a --filter "name=crawl4ai" --format "{{.Names}}" 2>/dev/null || echo "")
-if [[ -n "$CONTAINERS" ]]; then
-    echo "❌ Found remaining crawl4ai containers: $CONTAINERS"
-    exit 1
-fi
-
-NGINX=$(docker ps -a --filter "name=nginx" --format "{{.Names}}" 2>/dev/null || echo "")
-if [[ -n "$NGINX" ]]; then
-    echo "⚠️  Warning: Nginx container still exists: $NGINX"
-fi
-
-REDIS=$(docker ps -a --filter "name=redis" --format "{{.Names}}" 2>/dev/null || echo "")
-if [[ -n "$REDIS" ]]; then
-    echo "⚠️  Warning: Redis container still exists: $REDIS"
-fi
-
-# Verify port is free
-if curl -s http://localhost:11235/health > /dev/null 2>&1; then
-    echo "❌ Port 11235 still in use after cleanup"
-    exit 1
-fi
-
-echo ""
-echo "✅ Test passed: All Crawl4AI resources properly cleaned up"

deploy/docker/tests/cli/resource/test_05_long_running.sh

@@ -1,99 +0,0 @@
-#!/bin/bash
-# Test: Long-running stability test (5 minutes)
-# Expected: Server remains stable over extended period
-
-set -e
-
-echo "=== Test: Long-Running Stability (5 minutes) ==="
-echo ""
-
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
-source "$PROJECT_ROOT/venv/bin/activate"
-
-# Cleanup
-crwl server stop 2>/dev/null || true
-sleep 2
-
-# Start server
-echo "Starting server with 2 replicas..."
-crwl server start --replicas 2 >/dev/null 2>&1
-sleep 10
-
-# Get start time
-START_TIME=$(date +%s)
-DURATION=300  # 5 minutes in seconds
-REQUEST_COUNT=0
-ERROR_COUNT=0
-
-echo ""
-echo "Running stability test for 5 minutes..."
-echo "Making periodic requests every 10 seconds..."
-echo ""
-
-while true; do
-    CURRENT_TIME=$(date +%s)
-    ELAPSED=$((CURRENT_TIME - START_TIME))
-
-    if [[ $ELAPSED -ge $DURATION ]]; then
-        break
-    fi
-
-    REMAINING=$((DURATION - ELAPSED))
-    echo "[$ELAPSED/$DURATION seconds] Remaining: ${REMAINING}s, Requests: $REQUEST_COUNT, Errors: $ERROR_COUNT"
-
-    # Make a request
-    if curl -s -X POST http://localhost:11235/crawl \
-        -H "Content-Type: application/json" \
-        -d '{"urls": ["https://httpbin.org/html"], "crawler_config": {}}' > /dev/null 2>&1; then
-        REQUEST_COUNT=$((REQUEST_COUNT + 1))
-    else
-        ERROR_COUNT=$((ERROR_COUNT + 1))
-        echo "  ⚠️  Request failed"
-    fi
-
-    # Check health every 30 seconds
-    if [[ $((ELAPSED % 30)) -eq 0 ]]; then
-        HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
-        if [[ "$HEALTH" != "ok" ]]; then
-            echo "  ❌ Health check failed!"
-            ERROR_COUNT=$((ERROR_COUNT + 1))
-        fi
-
-        # Get memory stats
-        MEM=$(curl -s http://localhost:11235/monitor/health | jq -r '.container.memory_percent' 2>/dev/null || echo "N/A")
-        echo "  Memory: ${MEM}%"
-    fi
-
-    sleep 10
-done
-
-echo ""
-echo "Test duration completed!"
-echo "Total requests: $REQUEST_COUNT"
-echo "Total errors: $ERROR_COUNT"
-
-# Get final stats
-echo ""
-echo "Final statistics:"
-curl -s http://localhost:11235/monitor/endpoints/stats | jq '.' 2>/dev/null || echo "No stats available"
-
-# Verify error rate is acceptable (<10%)
-ERROR_RATE=$(echo "scale=2; $ERROR_COUNT * 100 / $REQUEST_COUNT" | bc -l 2>/dev/null || echo "0")
-echo ""
-echo "Error rate: ${ERROR_RATE}%"
-
-# Cleanup
-echo ""
-echo "Cleaning up..."
-crwl server stop >/dev/null 2>&1
-
-# Check error rate
-ERROR_OK=$(echo "$ERROR_RATE < 10" | bc -l 2>/dev/null || echo "1")
-if [[ "$ERROR_OK" != "1" ]]; then
-    echo "❌ Error rate too high: ${ERROR_RATE}%"
-    exit 1
-fi
-
-echo ""
-echo "✅ Test passed: Server remained stable over 5 minutes"
-echo "   Requests: $REQUEST_COUNT, Errors: $ERROR_COUNT, Error rate: ${ERROR_RATE}%"

deploy/docker/tests/cli/run_tests.sh

@@ -1,200 +0,0 @@
-#!/bin/bash
-# Master Test Runner for Crawl4AI CLI E2E Tests
-# Usage: ./run_tests.sh [category] [test_number]
-#   category: basic|advanced|resource|dashboard|edge|all (default: all)
-#   test_number: specific test number to run (optional)
-
-set -e
-
-# Color codes for output
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-BLUE='\033[0;34m'
-NC='\033[0m' # No Color
-
-# Test counters
-TOTAL_TESTS=0
-PASSED_TESTS=0
-FAILED_TESTS=0
-SKIPPED_TESTS=0
-
-# Get script directory
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-# Print header
-print_header() {
-    echo ""
-    echo "=========================================="
-    echo "$1"
-    echo "=========================================="
-    echo ""
-}
-
-# Print test result
-print_result() {
-    local test_name=$1
-    local result=$2
-
-    if [[ "$result" == "PASS" ]]; then
-        echo -e "${GREEN}✅ PASS${NC}: $test_name"
-        PASSED_TESTS=$((PASSED_TESTS + 1))
-    elif [[ "$result" == "FAIL" ]]; then
-        echo -e "${RED}❌ FAIL${NC}: $test_name"
-        FAILED_TESTS=$((FAILED_TESTS + 1))
-    elif [[ "$result" == "SKIP" ]]; then
-        echo -e "${YELLOW}⏭️  SKIP${NC}: $test_name"
-        SKIPPED_TESTS=$((SKIPPED_TESTS + 1))
-    fi
-}
-
-# Run a single test
-run_test() {
-    local test_path=$1
-    local test_name=$(basename "$test_path")
-
-    echo ""
-    echo -e "${BLUE}Running:${NC} $test_name"
-    echo "----------------------------------------"
-
-    TOTAL_TESTS=$((TOTAL_TESTS + 1))
-
-    if bash "$test_path"; then
-        print_result "$test_name" "PASS"
-        return 0
-    else
-        print_result "$test_name" "FAIL"
-        return 1
-    fi
-}
-
-# Run Python test
-run_python_test() {
-    local test_path=$1
-    local test_name=$(basename "$test_path")
-
-    echo ""
-    echo -e "${BLUE}Running:${NC} $test_name"
-    echo "----------------------------------------"
-
-    TOTAL_TESTS=$((TOTAL_TESTS + 1))
-
-    if python "$test_path"; then
-        print_result "$test_name" "PASS"
-        return 0
-    else
-        print_result "$test_name" "FAIL"
-        return 1
-    fi
-}
-
-# Run tests in a category
-run_category() {
-    local category=$1
-    local test_number=$2
-    local category_dir="$SCRIPT_DIR/$category"
-
-    if [[ ! -d "$category_dir" ]]; then
-        echo -e "${RED}Error:${NC} Category '$category' not found"
-        return 1
-    fi
-
-    print_header "Running $category tests"
-
-    if [[ -n "$test_number" ]]; then
-        # Run specific test
-        local test_file=$(find "$category_dir" -name "*${test_number}*.sh" | head -n 1)
-        if [[ -z "$test_file" ]]; then
-            echo -e "${RED}Error:${NC} Test $test_number not found in $category"
-            return 1
-        fi
-        run_test "$test_file"
-    else
-        # Run all tests in category
-        if [[ "$category" == "dashboard" ]]; then
-            # Dashboard tests are Python
-            for test_file in "$category_dir"/*.py; do
-                [[ -f "$test_file" ]] || continue
-                run_python_test "$test_file" || true
-            done
-        else
-            # Shell script tests
-            for test_file in "$category_dir"/*.sh; do
-                [[ -f "$test_file" ]] || continue
-                run_test "$test_file" || true
-            done
-        fi
-    fi
-}
-
-# Print summary
-print_summary() {
-    echo ""
-    echo "=========================================="
-    echo "Test Summary"
-    echo "=========================================="
-    echo -e "Total:   $TOTAL_TESTS"
-    echo -e "${GREEN}Passed:  $PASSED_TESTS${NC}"
-    echo -e "${RED}Failed:  $FAILED_TESTS${NC}"
-    echo -e "${YELLOW}Skipped: $SKIPPED_TESTS${NC}"
-    echo ""
-
-    if [[ $FAILED_TESTS -eq 0 ]]; then
-        echo -e "${GREEN}✅ All tests passed!${NC}"
-        return 0
-    else
-        echo -e "${RED}❌ Some tests failed${NC}"
-        return 1
-    fi
-}
-
-# Main execution
-main() {
-    local category=${1:-all}
-    local test_number=$2
-
-    # Activate virtual environment
-    if [[ -f "venv/bin/activate" ]]; then
-        source venv/bin/activate
-    else
-        echo -e "${YELLOW}Warning:${NC} venv not found, some tests may fail"
-    fi
-
-    print_header "Crawl4AI CLI E2E Test Suite"
-
-    if [[ "$category" == "all" ]]; then
-        # Run all categories
-        for cat in basic advanced resource edge; do
-            run_category "$cat" || true
-        done
-        # Dashboard tests separately (can be slow)
-        echo ""
-        echo -e "${YELLOW}Note:${NC} Dashboard tests can be run separately with: ./run_tests.sh dashboard"
-    else
-        run_category "$category" "$test_number"
-    fi
-
-    print_summary
-}
-
-# Show usage
-if [[ "$1" == "-h" || "$1" == "--help" ]]; then
-    echo "Usage: $0 [category] [test_number]"
-    echo ""
-    echo "Categories:"
-    echo "  basic      - Basic CLI operations (8 tests)"
-    echo "  advanced   - Advanced features (8 tests)"
-    echo "  resource   - Resource monitoring and stress tests (5 tests)"
-    echo "  dashboard  - Dashboard UI tests with Playwright (1 test)"
-    echo "  edge       - Edge cases and error handling (10 tests)"
-    echo "  all        - Run all tests except dashboard (default)"
-    echo ""
-    echo "Examples:"
-    echo "  $0                    # Run all tests"
-    echo "  $0 basic              # Run all basic tests"
-    echo "  $0 basic 01           # Run test_01 from basic"
-    echo "  $0 dashboard          # Run dashboard UI test"
-    exit 0
-fi
-
-main "$@"

deploy/docker/utils.py

@@ -203,51 +203,4 @@ def get_container_memory_percent() -> float:
     except:
         # Non-container or unsupported: fallback to host
         import psutil
-        return psutil.virtual_memory().percent
+        return psutil.virtual_memory().percent
-
-
-def get_container_id() -> str:
-    """Get current container ID (hostname in Docker)."""
-    import socket
-    return socket.gethostname()
-
-
-def detect_deployment_mode() -> tuple[str, list[dict]]:
-    """Detect if running in single/swarm/compose mode and get container list.
-
-    Returns:
-        (mode, containers) where mode is "single"|"swarm"|"compose"
-        containers is list of {id, hostname, healthy}
-    """
-    import socket
-    my_hostname = socket.gethostname()
-
-    # Check if we're behind nginx (Compose mode indicator)
-    # In Compose, service name resolves to multiple IPs
-    try:
-        import socket as sock
-        # Try to resolve "crawl4ai" service name (Compose service)
-        try:
-            addrs = sock.getaddrinfo("crawl4ai", None)
-            unique_ips = set(addr[4][0] for addr in addrs)
-            if len(unique_ips) > 1:
-                # Multiple IPs = Compose with replicas
-                containers = [
-                    {"id": f"container-{i+1}", "hostname": f"crawl4ai-{i+1}", "healthy": True}
-                    for i in range(len(unique_ips))
-                ]
-                return "compose", containers
-        except:
-            pass
-
-        # Check for Swarm mode (TODO: needs swarm-specific detection)
-        # For now, if hostname pattern matches swarm, detect it
-        if "." in my_hostname and len(my_hostname.split(".")) > 2:
-            # Swarm hostname format: service.slot.task_id
-            return "swarm", [{"id": my_hostname, "hostname": my_hostname, "healthy": True}]
-
-    except:
-        pass
-
-    # Default: single container
-    return "single", [{"id": my_hostname, "hostname": my_hostname, "healthy": True}]

deploy/docker/webhook.py

@@ -1,159 +0,0 @@
-"""
-Webhook delivery service for Crawl4AI.
-
-This module provides webhook notification functionality with exponential backoff retry logic.
-"""
-import asyncio
-import httpx
-import logging
-from typing import Dict, Optional
-from datetime import datetime, timezone
-
-logger = logging.getLogger(__name__)
-
-
-class WebhookDeliveryService:
-    """Handles webhook delivery with exponential backoff retry logic."""
-
-    def __init__(self, config: Dict):
-        """
-        Initialize the webhook delivery service.
-
-        Args:
-            config: Application configuration dictionary containing webhook settings
-        """
-        self.config = config.get("webhooks", {})
-        self.max_attempts = self.config.get("retry", {}).get("max_attempts", 5)
-        self.initial_delay = self.config.get("retry", {}).get("initial_delay_ms", 1000) / 1000
-        self.max_delay = self.config.get("retry", {}).get("max_delay_ms", 32000) / 1000
-        self.timeout = self.config.get("retry", {}).get("timeout_ms", 30000) / 1000
-
-    async def send_webhook(
-        self,
-        webhook_url: str,
-        payload: Dict,
-        headers: Optional[Dict[str, str]] = None
-    ) -> bool:
-        """
-        Send webhook with exponential backoff retry logic.
-
-        Args:
-            webhook_url: The URL to send the webhook to
-            payload: The JSON payload to send
-            headers: Optional custom headers
-
-        Returns:
-            bool: True if delivered successfully, False otherwise
-        """
-        default_headers = self.config.get("headers", {})
-        merged_headers = {**default_headers, **(headers or {})}
-        merged_headers["Content-Type"] = "application/json"
-
-        async with httpx.AsyncClient(timeout=self.timeout) as client:
-            for attempt in range(self.max_attempts):
-                try:
-                    logger.info(
-                        f"Sending webhook (attempt {attempt + 1}/{self.max_attempts}) to {webhook_url}"
-                    )
-
-                    response = await client.post(
-                        webhook_url,
-                        json=payload,
-                        headers=merged_headers
-                    )
-
-                    # Success or client error (don't retry client errors)
-                    if response.status_code < 500:
-                        if 200 <= response.status_code < 300:
-                            logger.info(f"Webhook delivered successfully to {webhook_url}")
-                            return True
-                        else:
-                            logger.warning(
-                                f"Webhook rejected with status {response.status_code}: {response.text[:200]}"
-                            )
-                            return False  # Client error - don't retry
-
-                    # Server error - retry with backoff
-                    logger.warning(
-                        f"Webhook failed with status {response.status_code}, will retry"
-                    )
-
-                except httpx.TimeoutException as exc:
-                    logger.error(f"Webhook timeout (attempt {attempt + 1}): {exc}")
-                except httpx.RequestError as exc:
-                    logger.error(f"Webhook request error (attempt {attempt + 1}): {exc}")
-                except Exception as exc:
-                    logger.error(f"Webhook delivery error (attempt {attempt + 1}): {exc}")
-
-                # Calculate exponential backoff delay
-                if attempt < self.max_attempts - 1:
-                    delay = min(self.initial_delay * (2 ** attempt), self.max_delay)
-                    logger.info(f"Retrying in {delay}s...")
-                    await asyncio.sleep(delay)
-
-        logger.error(
-            f"Webhook delivery failed after {self.max_attempts} attempts to {webhook_url}"
-        )
-        return False
-
-    async def notify_job_completion(
-        self,
-        task_id: str,
-        task_type: str,
-        status: str,
-        urls: list,
-        webhook_config: Optional[Dict],
-        result: Optional[Dict] = None,
-        error: Optional[str] = None
-    ):
-        """
-        Notify webhook of job completion.
-
-        Args:
-            task_id: The task identifier
-            task_type: Type of task (e.g., "crawl", "llm_extraction")
-            status: Task status ("completed" or "failed")
-            urls: List of URLs that were crawled
-            webhook_config: Webhook configuration from the job request
-            result: Optional crawl result data
-            error: Optional error message if failed
-        """
-        # Determine webhook URL
-        webhook_url = None
-        data_in_payload = self.config.get("data_in_payload", False)
-        custom_headers = None
-
-        if webhook_config:
-            webhook_url = webhook_config.get("webhook_url")
-            data_in_payload = webhook_config.get("webhook_data_in_payload", data_in_payload)
-            custom_headers = webhook_config.get("webhook_headers")
-
-        if not webhook_url:
-            webhook_url = self.config.get("default_url")
-
-        if not webhook_url:
-            logger.debug("No webhook URL configured, skipping notification")
-            return
-
-        # Check if webhooks are enabled
-        if not self.config.get("enabled", True):
-            logger.debug("Webhooks are disabled, skipping notification")
-            return
-
-        # Build payload
-        payload = {
-            "task_id": task_id,
-            "task_type": task_type,
-            "status": status,
-            "timestamp": datetime.now(timezone.utc).isoformat(),
-            "urls": urls
-        }
-
-        if error:
-            payload["error"] = error
-
-        if data_in_payload and result:
-            payload["data"] = result
-
-        # Send webhook (fire and forget - don't block on completion)
-        await self.send_webhook(webhook_url, payload, custom_headers)

deploy/installer/QUICKSTART.md

@@ -1,147 +0,0 @@
-# Crawl4AI cnode - Quick Start Cheat Sheet
-
-Fast reference for getting started with cnode.
-
----
-
-## 📥 Install
-
-```bash
-# Install cnode
-curl -sSL https://raw.githubusercontent.com/unclecode/crawl4ai/main/deploy/installer/install-cnode.sh | bash
-```
-
----
-
-## 🚀 Launch Cluster
-
-```bash
-# Single server (development)
-cnode start
-
-# Production cluster with 5 replicas
-cnode start --replicas 5
-
-# Custom port
-cnode start --replicas 3 --port 8080
-```
-
----
-
-## 📊 Check Status
-
-```bash
-# View server status
-cnode status
-
-# View logs
-cnode logs -f
-```
-
----
-
-## ⚙️ Scale Cluster
-
-```bash
-# Scale to 10 replicas (live, no downtime)
-cnode scale 10
-
-# Scale down to 2
-cnode scale 2
-```
-
----
-
-## 🔄 Restart/Stop
-
-```bash
-# Restart server
-cnode restart
-
-# Stop server
-cnode stop
-```
-
----
-
-## 🌐 Test the API
-
-```bash
-# Simple test - crawl example.com
-curl -X POST http://localhost:11235/crawl \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://example.com"],
-    "priority": 10
-  }'
-
-# Pretty print with jq
-curl -X POST http://localhost:11235/crawl \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://example.com"],
-    "priority": 10
-  }' | jq '.result.markdown' -r
-
-# Health check
-curl http://localhost:11235/health
-```
-
----
-
-## 📱 Monitor Dashboard
-
-```bash
-# Open in browser
-open http://localhost:11235/monitor
-
-# Or playground
-open http://localhost:11235/playground
-```
-
----
-
-## 🐍 Python Example
-
-```python
-import requests
-
-response = requests.post(
-    "http://localhost:11235/crawl",
-    json={
-        "urls": ["https://example.com"],
-        "priority": 10
-    }
-)
-
-result = response.json()
-print(result['result']['markdown'])
-```
-
----
-
-## 🎯 Common Commands
-
-| Command | Description |
-|---------|-------------|
-| `cnode start` | Start server |
-| `cnode start -r 5` | Start with 5 replicas |
-| `cnode status` | Check status |
-| `cnode scale 10` | Scale to 10 replicas |
-| `cnode logs -f` | Follow logs |
-| `cnode restart` | Restart server |
-| `cnode stop` | Stop server |
-| `cnode --help` | Show all commands |
-
----
-
-## 📚 Full Documentation
-
-- **User Guide:** `deploy/installer/USER_GUIDE.md`
-- **Developer Docs:** `deploy/installer/README.md`
-- **Docker Guide:** `deploy/docker/README.md`
-- **Agent Context:** `deploy/docker/AGENT.md`
-
----
-
-**That's it!** You're ready to crawl at scale 🚀

deploy/installer/README.md

@@ -1,345 +0,0 @@
-# Crawl4AI Node Manager (cnode) - Installation & Distribution
-
-This directory contains the standalone `cnode` package and installation scripts for managing Crawl4AI Docker server instances.
-
-## Overview
-
-`cnode` is a fast, lightweight CLI tool for managing Crawl4AI Docker servers. It provides:
-- One-command deployment with automatic scaling
-- Single container for development (N=1)
-- Docker Swarm for production with built-in load balancing (N>1)
-- Docker Compose + Nginx as fallback (N>1)
-
-## Directory Structure
-
-```
-deploy/installer/
-├── README.md                 # This file
-├── cnode_pkg/                # Standalone Python package
-│   ├── __init__.py           # Package marker
-│   ├── cli.py                # CLI interface (commands)
-│   ├── server_manager.py     # Docker orchestration logic
-│   └── requirements.txt      # Python dependencies
-├── install-cnode.sh          # Local installation script
-├── install-cnode.sh                 # Remote installation script (for users)
-└── releases/                 # Release artifacts for distribution
-```
-
-## Installation
-
-### For Users (Remote Installation)
-
-Users can install `cnode` directly from the web:
-
-```bash
-# Install from GitHub/website
-curl -sSL https://crawl4ai.com/install-cnode.sh | bash
-
-# Or with wget
-wget -qO- https://crawl4ai.com/install-cnode.sh | bash
-```
-
-### For Local Testing
-
-Test the installation locally:
-
-```bash
-cd deploy/installer
-./install-cnode.sh
-```
-
-## Package Contents
-
-### `cnode_pkg/` - Python Package
-
-This is a self-contained Python package with:
-
-- **`cli.py`**: Click-based CLI with all commands (start, stop, status, scale, logs, cleanup, restart)
-- **`server_manager.py`**: Core Docker orchestration logic
-- **`requirements.txt`**: Dependencies (click, rich, anyio, pyyaml)
-- **`__init__.py`**: Package initialization
-
-### Installation Script
-
-**`install-cnode.sh`** does the following:
-1. Checks for Python 3.8+ and pip
-2. Checks for Docker (warns if not found)
-3. Installs Python dependencies
-4. Copies `cnode_pkg/` to `/usr/local/lib/cnode/`
-5. Creates wrapper script at `/usr/local/bin/cnode`
-6. Verifies installation
-
-### Wrapper Script
-
-Created at `/usr/local/bin/cnode`:
-
-```bash
-#!/usr/bin/env bash
-set -e
-
-# Find Python
-if command -v python3 &> /dev/null; then
-    PYTHON_CMD="python3"
-elif command -v python &> /dev/null; then
-    PYTHON_CMD="python"
-else
-    echo "Error: Python 3.8+ required" >&2
-    exit 1
-fi
-
-# Run cnode
-export PYTHONPATH="/usr/local/lib/cnode:$PYTHONPATH"
-exec $PYTHON_CMD -m cnode_pkg.cli "$@"
-```
-
-## Performance
-
-**Blazing Fast Startup:**
-- **~0.1 seconds** to launch
-- 49x faster than compiled binary alternatives
-- Minimal overhead, maximum responsiveness
-
-## Requirements
-
-### User Requirements
-- Python 3.8 or higher
-- pip (Python package manager)
-- Docker (for running servers)
-
-### Dependencies (Auto-installed)
-- click >= 8.0.0 (CLI framework)
-- rich >= 13.0.0 (Terminal formatting)
-- anyio >= 3.0.0 (Async I/O)
-- pyyaml >= 6.0.0 (YAML parsing)
-
-## Usage
-
-After installation:
-
-```bash
-# Quick start
-cnode start                    # Single container on port 11235
-cnode start --replicas 5       # 5-replica cluster
-cnode status                   # Check server status
-cnode logs -f                  # Follow logs
-cnode scale 10                 # Scale to 10 replicas
-cnode stop                     # Stop server
-
-# Get help
-cnode --help
-cnode start --help
-```
-
-## Development Workflow
-
-### Making Changes
-
-1. **Edit source code** in `deploy/docker/`:
-   ```bash
-   vim deploy/docker/cnode_cli.py
-   vim deploy/docker/server_manager.py
-   ```
-
-2. **Update package** by copying to installer:
-   ```bash
-   # Copy CLI
-   cp deploy/docker/cnode_cli.py deploy/installer/cnode_pkg/cli.py
-
-   # Fix imports (deploy.docker → cnode_pkg)
-   sed -i 's/from deploy\.docker\./from cnode_pkg./g' deploy/installer/cnode_pkg/cli.py
-
-   # Copy server manager
-   cp deploy/docker/server_manager.py deploy/installer/cnode_pkg/server_manager.py
-   ```
-
-3. **Test locally**:
-   ```bash
-   cd deploy/installer
-   ./install-cnode.sh
-   cnode --help
-   ```
-
-4. **Commit both**:
-   ```bash
-   git add deploy/docker/cnode_cli.py
-   git add deploy/installer/cnode_pkg/cli.py
-   git commit -m "Update cnode: [description]"
-   ```
-
-### Creating a Release
-
-1. **Tag the release**:
-   ```bash
-   git tag -a v1.0.0 -m "Release v1.0.0"
-   git push origin v1.0.0
-   ```
-
-2. **Package for distribution**:
-   ```bash
-   cd deploy/installer
-   tar -czf releases/cnode-v1.0.0.tar.gz cnode_pkg/ install-cnode.sh
-   ```
-
-3. **Create GitHub release**:
-   ```bash
-   gh release create v1.0.0 \
-     releases/cnode-v1.0.0.tar.gz \
-     --title "cnode v1.0.0" \
-     --notes "Release notes here"
-   ```
-
-4. **Update deployment script** (if needed):
-   - Update `install-cnode.sh` with new version/URL
-   - Upload to hosting (e.g., `https://crawl4ai.com/install-cnode.sh`)
-
-## Deployment
-
-### Remote Installation Script
-
-The `install-cnode.sh` script is meant to be hosted at a public URL for user installation:
-
-```bash
-# Upload to your server
-scp install-cnode.sh user@crawl4ai.com:/var/www/html/install-cnode.sh
-
-# Or use GitHub raw URL
-https://raw.githubusercontent.com/unclecode/crawl4ai/main/deploy/installer/install-cnode.sh
-```
-
-Users can then install with:
-```bash
-curl -sSL https://crawl4ai.com/install-cnode.sh | bash
-```
-
-## Backward Compatibility
-
-The main Crawl4AI CLI (`crwl`) includes a redirect for backward compatibility:
-
-```bash
-# These work identically:
-crwl server start --replicas 3
-cnode start --replicas 3
-
-# All subcommands redirect:
-crwl server status  → cnode status
-crwl server stop    → cnode stop
-crwl server scale 5 → cnode scale 5
-crwl server logs -f → cnode logs -f
-```
-
-This ensures existing scripts continue working while users migrate to `cnode`.
-
-## Uninstallation
-
-To remove cnode:
-
-```bash
-# Remove command
-sudo rm /usr/local/bin/cnode
-
-# Remove package
-sudo rm -rf /usr/local/lib/cnode
-
-# (Optional) Uninstall dependencies
-pip uninstall click rich anyio pyyaml
-```
-
-## Troubleshooting
-
-### Python Not Found
-```bash
-# Install Python 3.8+
-# macOS: brew install python3
-# Ubuntu: sudo apt install python3 python3-pip
-# RHEL/CentOS: sudo yum install python3 python3-pip
-```
-
-### Permission Denied
-```bash
-# Run installer with sudo
-sudo ./install-cnode.sh
-
-# Or change install location
-INSTALL_DIR=$HOME/.local/bin ./install-cnode.sh
-```
-
-### Command Not Found After Install
-```bash
-# Add to PATH in ~/.bashrc or ~/.zshrc
-export PATH="/usr/local/bin:$PATH"
-
-# Reload shell
-source ~/.bashrc  # or source ~/.zshrc
-```
-
-### Dependencies Install Failed
-```bash
-# Install manually
-pip install --user click rich anyio pyyaml
-
-# Or with break-system-packages (if needed)
-pip install --user --break-system-packages click rich anyio pyyaml
-```
-
-### Docker Not Running
-```bash
-# macOS: Start Docker Desktop
-# Linux: sudo systemctl start docker
-
-# Check Docker
-docker --version
-docker ps
-```
-
-## Architecture
-
-### Component Flow
-
-```
-User runs: cnode start
-         ↓
-/usr/local/bin/cnode (wrapper script)
-         ↓
-Finds python3 executable
-         ↓
-Sets PYTHONPATH=/usr/local/lib/cnode
-         ↓
-python3 -m cnode_pkg.cli start
-         ↓
-cli.py → start_cmd()
-         ↓
-server_manager.py → ServerManager.start()
-         ↓
-Docker orchestration (single/swarm/compose)
-         ↓
-Server running!
-```
-
-### Why Python Wrapper vs Binary?
-
-We chose a Python wrapper over compiled binaries (PyInstaller) because:
-
-| Metric | Python Wrapper | PyInstaller Binary |
-|--------|---------------|-------------------|
-| Startup time | **0.1s** | 4.7s |
-| Size | ~50KB wrapper | 8.8MB |
-| Updates | Easy (just copy files) | Rebuild required |
-| Dependencies | Python 3.8+ | None |
-| Platform | Any with Python | OS-specific builds |
-
-Since users running Crawl4AI already have Python, the wrapper is the clear winner.
-
-## Support
-
-For issues or questions:
-- GitHub Issues: https://github.com/unclecode/crawl4ai/issues
-- Documentation: https://docs.crawl4ai.com
-- Discord: https://discord.gg/crawl4ai
-
-## Version History
-
-- **v1.0.0**: Initial release with Python wrapper approach
-  - Fast startup (~0.1s)
-  - Supports single container, Docker Swarm, and Compose modes
-  - Auto-scaling and load balancing
-  - Real-time monitoring and logs

deploy/installer/USER_GUIDE.md

@@ -1,676 +0,0 @@
-# Crawl4AI Node Manager (cnode) - User Guide 🚀
-
-Self-host your own Crawl4AI server cluster with one command. Scale from development to production effortlessly.
-
-## Table of Contents
-- [What is cnode?](#what-is-cnode)
-- [Quick Start](#quick-start)
-- [Installation](#installation)
-- [Basic Usage](#basic-usage)
-- [Scaling & Production](#scaling--production)
-- [Monitoring Dashboard](#monitoring-dashboard)
-- [Using the API](#using-the-api)
-- [Management Commands](#management-commands)
-- [Troubleshooting](#troubleshooting)
-- [Advanced Topics](#advanced-topics)
-
----
-
-## What is cnode?
-
-**cnode** (Crawl4AI Node Manager) is a CLI tool that manages Crawl4AI Docker server instances with automatic scaling and load balancing.
-
-### Key Features
-
-✅ **One-Command Deployment** - Start a server or cluster instantly
-✅ **Automatic Scaling** - Single container or multi-replica cluster
-✅ **Built-in Load Balancing** - Docker Swarm or Nginx (auto-detected)
-✅ **Real-time Monitoring** - Beautiful web dashboard
-✅ **Zero Configuration** - Works out of the box
-✅ **Production Ready** - Auto-scaling, health checks, rolling updates
-
-### Architecture Modes
-
-| Replicas | Mode | Load Balancer | Use Case |
-|----------|------|---------------|----------|
-| 1 | Single Container | None | Development, testing |
-| 2+ | Docker Swarm | Built-in | Production (if Swarm available) |
-| 2+ | Docker Compose | Nginx | Production (fallback) |
-
----
-
-## Quick Start
-
-### 1. Install cnode
-
-```bash
-# One-line installation
-curl -sSL https://crawl4ai.com/install-cnode.sh | bash
-```
-
-**Requirements:**
-- Python 3.8+
-- Docker
-- Git
-
-### 2. Start Your First Server
-
-```bash
-# Start single development server
-cnode start
-
-# Or start a production cluster with 5 replicas
-cnode start --replicas 5
-```
-
-That's it! Your server is running at **http://localhost:11235** 🎉
-
----
-
-## Installation
-
-### Method 1: Quick Install (Recommended)
-
-```bash
-# From crawl4ai.com (when hosted)
-curl -sSL https://crawl4ai.com/install-cnode.sh | bash
-
-# Or directly from GitHub
-curl -sSL https://raw.githubusercontent.com/unclecode/crawl4ai/main/deploy/installer/install-cnode.sh | bash
-```
-
-### Method 2: Clone Repository (For Development)
-
-```bash
-# Clone the repository
-git clone https://github.com/unclecode/crawl4ai.git
-cd crawl4ai/deploy/installer
-
-# Run installer
-./install-cnode.sh
-```
-
-### Method 3: Custom Location
-
-```bash
-# Install to custom directory (using GitHub raw URL)
-INSTALL_DIR=$HOME/.local/bin curl -sSL https://raw.githubusercontent.com/unclecode/crawl4ai/main/deploy/installer/install-cnode.sh | bash
-
-# Add to PATH
-export PATH="$HOME/.local/bin:$PATH"
-```
-
-### Verify Installation
-
-```bash
-cnode --help
-```
-
----
-
-## Basic Usage
-
-### Start Server
-
-```bash
-# Development server (1 replica)
-cnode start
-
-# Production cluster (5 replicas with auto-scaling)
-cnode start --replicas 5
-
-# Custom port
-cnode start --port 8080
-
-# Specific Docker image
-cnode start --image unclecode/crawl4ai:0.7.0
-```
-
-### Check Status
-
-```bash
-cnode status
-```
-
-**Example Output:**
-```
-╭─────────────────── Crawl4AI Server Status ───────────────────╮
-│ Status     │ 🟢 Running                                      │
-│ Mode       │ swarm                                           │
-│ Replicas   │ 5                                               │
-│ Port       │ 11235                                           │
-│ Image      │ unclecode/crawl4ai:latest                       │
-│ Uptime     │ 2 hours 34 minutes                              │
-│ Started    │ 2025-10-21 14:30:00                            │
-╰─────────────────────────────────────────────────────────────╯
-
-✓ Server is healthy
-Access: http://localhost:11235
-```
-
-### View Logs
-
-```bash
-# Show last 100 lines
-cnode logs
-
-# Follow logs in real-time
-cnode logs -f
-
-# Show last 500 lines
-cnode logs --tail 500
-```
-
-### Stop Server
-
-```bash
-# Stop server (keeps data)
-cnode stop
-
-# Stop and remove all data
-cnode stop --remove-volumes
-```
-
----
-
-## Scaling & Production
-
-### Scale Your Cluster
-
-```bash
-# Scale to 10 replicas (live, no downtime)
-cnode scale 10
-
-# Scale down to 2 replicas
-cnode scale 2
-```
-
-**Note:** Scaling is live for Swarm/Compose modes. Single container mode requires restart.
-
-### Production Deployment
-
-```bash
-# Start production cluster
-cnode start --replicas 5 --port 11235
-
-# Verify health
-curl http://localhost:11235/health
-
-# Monitor performance
-cnode logs -f
-```
-
-### Restart Server
-
-```bash
-# Restart with same configuration
-cnode restart
-
-# Restart with new replica count
-cnode restart --replicas 10
-```
-
----
-
-## Monitoring Dashboard
-
-### Access the Dashboard
-
-Once your server is running, access the real-time monitoring dashboard:
-
-```bash
-# Dashboard URL
-http://localhost:11235/monitor
-```
-
-### Dashboard Features
-
-📊 **Real-time Metrics**
-- Requests per second
-- Active connections
-- Response times
-- Error rates
-
-📈 **Performance Graphs**
-- CPU usage
-- Memory consumption
-- Request latency
-- Throughput
-
-🔍 **System Health**
-- Container status
-- Replica health
-- Load distribution
-- Resource utilization
-
-![Monitor Dashboard](https://crawl4ai.com/images/monitor-dashboard.png)
-
-### API Health Endpoint
-
-```bash
-# Quick health check
-curl http://localhost:11235/health
-
-# Response
-{
-  "status": "healthy",
-  "version": "1.0.0",
-  "uptime": 9876,
-  "containers": 5
-}
-```
-
----
-
-## Using the API
-
-### Interactive Playground
-
-Test the API interactively:
-
-```
-http://localhost:11235/playground
-```
-
-### Basic Crawl Example
-
-**Python:**
-
-```python
-import requests
-
-# Simple crawl
-response = requests.post(
-    "http://localhost:11235/crawl",
-    json={
-        "urls": ["https://example.com"],
-        "browser_config": {
-            "type": "BrowserConfig",
-            "params": {"headless": True}
-        },
-        "crawler_config": {
-            "type": "CrawlerRunConfig",
-            "params": {"cache_mode": "bypass"}
-        }
-    }
-)
-
-result = response.json()
-print(f"Title: {result['result']['metadata']['title']}")
-print(f"Content: {result['result']['markdown'][:200]}...")
-```
-
-**cURL:**
-
-```bash
-curl -X POST http://localhost:11235/crawl \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": ["https://example.com"],
-    "browser_config": {
-      "type": "BrowserConfig",
-      "params": {"headless": true}
-    },
-    "crawler_config": {
-      "type": "CrawlerRunConfig",
-      "params": {"cache_mode": "bypass"}
-    }
-  }'
-```
-
-**JavaScript (Node.js):**
-
-```javascript
-const axios = require('axios');
-
-async function crawl() {
-  const response = await axios.post('http://localhost:11235/crawl', {
-    urls: ['https://example.com'],
-    browser_config: {
-      type: 'BrowserConfig',
-      params: { headless: true }
-    },
-    crawler_config: {
-      type: 'CrawlerRunConfig',
-      params: { cache_mode: 'bypass' }
-    }
-  });
-
-  console.log('Title:', response.data.result.metadata.title);
-  console.log('Content:', response.data.result.markdown.substring(0, 200));
-}
-
-crawl();
-```
-
-### Advanced Examples
-
-**Extract with CSS Selectors:**
-
-```python
-import requests
-
-response = requests.post(
-    "http://localhost:11235/crawl",
-    json={
-        "urls": ["https://news.ycombinator.com"],
-        "browser_config": {
-            "type": "BrowserConfig",
-            "params": {"headless": True}
-        },
-        "crawler_config": {
-            "type": "CrawlerRunConfig",
-            "params": {
-                "extraction_strategy": {
-                    "type": "JsonCssExtractionStrategy",
-                    "params": {
-                        "schema": {
-                            "type": "dict",
-                            "value": {
-                                "baseSelector": ".athing",
-                                "fields": [
-                                    {"name": "title", "selector": ".titleline > a", "type": "text"},
-                                    {"name": "url", "selector": ".titleline > a", "type": "attribute", "attribute": "href"},
-                                    {"name": "points", "selector": ".score", "type": "text"}
-                                ]
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-)
-
-articles = response.json()['result']['extracted_content']
-for article in articles:
-    print(f"{article['title']} - {article['points']}")
-```
-
-**Streaming Multiple URLs:**
-
-```python
-import requests
-import json
-
-response = requests.post(
-    "http://localhost:11235/crawl/stream",
-    json={
-        "urls": [
-            "https://example.com",
-            "https://httpbin.org/html",
-            "https://python.org"
-        ],
-        "browser_config": {
-            "type": "BrowserConfig",
-            "params": {"headless": True}
-        },
-        "crawler_config": {
-            "type": "CrawlerRunConfig",
-            "params": {"stream": True}
-        }
-    },
-    stream=True
-)
-
-for line in response.iter_lines():
-    if line:
-        data = json.loads(line)
-        if data.get("status") == "completed":
-            break
-        print(f"Crawled: {data['url']} - Success: {data['success']}")
-```
-
-### Additional Endpoints
-
-**Screenshot:**
-```bash
-curl -X POST http://localhost:11235/screenshot \
-  -H "Content-Type: application/json" \
-  -d '{"url": "https://example.com"}' \
-  --output screenshot.png
-```
-
-**PDF Export:**
-```bash
-curl -X POST http://localhost:11235/pdf \
-  -H "Content-Type: application/json" \
-  -d '{"url": "https://example.com"}' \
-  --output page.pdf
-```
-
-**HTML Extraction:**
-```bash
-curl -X POST http://localhost:11235/html \
-  -H "Content-Type: application/json" \
-  -d '{"url": "https://example.com"}'
-```
-
----
-
-## Management Commands
-
-### All Available Commands
-
-```bash
-cnode --help              # Show help
-cnode start [OPTIONS]     # Start server
-cnode stop [OPTIONS]      # Stop server
-cnode status              # Show status
-cnode scale N             # Scale to N replicas
-cnode logs [OPTIONS]      # View logs
-cnode restart [OPTIONS]   # Restart server
-cnode cleanup [--force]   # Clean up resources
-```
-
-### Command Options
-
-**start:**
-```bash
---replicas, -r N      # Number of replicas (default: 1)
---mode MODE           # Deployment mode: auto, single, swarm, compose
---port, -p PORT       # External port (default: 11235)
---env-file FILE       # Environment file path
---image IMAGE         # Docker image (default: unclecode/crawl4ai:latest)
-```
-
-**stop:**
-```bash
---remove-volumes      # Remove persistent data (WARNING: deletes data)
-```
-
-**logs:**
-```bash
---follow, -f          # Follow log output (like tail -f)
---tail N              # Number of lines to show (default: 100)
-```
-
-**scale:**
-```bash
-N                     # Target replica count (minimum: 1)
-```
-
----
-
-## Troubleshooting
-
-### Server Won't Start
-
-```bash
-# Check Docker is running
-docker ps
-
-# Check port availability
-lsof -i :11235
-
-# Check logs for errors
-cnode logs
-```
-
-### High Memory Usage
-
-```bash
-# Check current status
-cnode status
-
-# Restart to clear memory
-cnode restart
-
-# Scale down if needed
-cnode scale 2
-```
-
-### Slow Response Times
-
-```bash
-# Scale up for better performance
-cnode scale 10
-
-# Check system resources
-docker stats
-```
-
-### Cannot Connect to API
-
-```bash
-# Verify server is running
-cnode status
-
-# Check firewall
-sudo ufw status
-
-# Test locally
-curl http://localhost:11235/health
-```
-
-### Clean Slate
-
-```bash
-# Complete cleanup and restart
-cnode cleanup --force
-cnode start --replicas 5
-```
-
----
-
-## Advanced Topics
-
-### Environment Variables
-
-Create `.env` file for API keys:
-
-```bash
-# .env file
-OPENAI_API_KEY=sk-your-key
-ANTHROPIC_API_KEY=your-key
-```
-
-Use with cnode:
-```bash
-cnode start --env-file .env --replicas 3
-```
-
-### Custom Docker Image
-
-```bash
-# Use specific version
-cnode start --image unclecode/crawl4ai:0.7.0-r1
-
-# Use custom registry
-cnode start --image myregistry.com/crawl4ai:custom
-```
-
-### Production Best Practices
-
-1. **Use Multiple Replicas**
-   ```bash
-   cnode start --replicas 5
-   ```
-
-2. **Monitor Regularly**
-   ```bash
-   # Set up monitoring cron
-   */5 * * * * cnode status | mail -s "Crawl4AI Status" admin@example.com
-   ```
-
-3. **Regular Log Rotation**
-   ```bash
-   cnode logs --tail 1000 > crawl4ai.log
-   cnode restart
-   ```
-
-4. **Resource Limits**
-   - Ensure adequate RAM (2GB per replica minimum)
-   - Monitor disk space for cached data
-   - Use SSD for better performance
-
-### Integration Examples
-
-**With Python App:**
-```python
-from crawl4ai.docker_client import Crawl4aiDockerClient
-
-async def main():
-    async with Crawl4aiDockerClient(base_url="http://localhost:11235") as client:
-        results = await client.crawl(["https://example.com"])
-        print(results[0].markdown)
-```
-
-**With Node.js:**
-```javascript
-const Crawl4AI = require('crawl4ai-client');
-const client = new Crawl4AI('http://localhost:11235');
-
-client.crawl('https://example.com')
-  .then(result => console.log(result.markdown));
-```
-
-**With REST API:**
-Any language with HTTP client support can use the API!
-
----
-
-## Getting Help
-
-### Resources
-
-- 📖 [Full Documentation](https://docs.crawl4ai.com)
-- 🐛 [Report Issues](https://github.com/unclecode/crawl4ai/issues)
-- 💬 [Discord Community](https://discord.gg/crawl4ai)
-- 📺 [Video Tutorials](https://youtube.com/@crawl4ai)
-
-### Common Questions
-
-**Q: How many replicas should I use?**
-A: Start with 1 for development. Use 3-5 for production. Scale based on load.
-
-**Q: What's the difference between Swarm and Compose mode?**
-A: Swarm has built-in load balancing (faster). Compose uses Nginx (fallback if Swarm unavailable).
-
-**Q: Can I run multiple cnode instances?**
-A: Yes! Use different ports: `cnode start --port 8080`
-
-**Q: How do I update to the latest version?**
-A: Pull new image: `cnode stop && docker pull unclecode/crawl4ai:latest && cnode start`
-
----
-
-## Summary
-
-You now know how to:
-- ✅ Install cnode with one command
-- ✅ Start and manage Crawl4AI servers
-- ✅ Scale from 1 to 100+ replicas
-- ✅ Monitor performance in real-time
-- ✅ Use the API from any language
-- ✅ Troubleshoot common issues
-
-**Ready to crawl at scale!** 🚀
-
-For detailed Docker configuration and advanced deployment options, see the [Docker Guide](../docker/README.md).
-
----
-
-**Happy Crawling!** 🕷️
-
-*Made with ❤️ by the Crawl4AI team*

deploy/installer/cnode_pkg/__init__.py

@@ -1,5 +0,0 @@
-"""
-Crawl4AI Node Manager (cnode) - Docker server orchestration CLI
-"""
-
-__version__ = "1.0.0"

deploy/installer/cnode_pkg/cli.py

@@ -1,492 +0,0 @@
-"""
-Crawl4AI Server CLI Commands
-
-Provides `cnode` command group for Docker orchestration.
-"""
-
-import click
-import anyio
-from rich.console import Console
-from rich.table import Table
-from rich.panel import Panel
-from rich.prompt import Confirm
-
-from cnode_pkg.server_manager import ServerManager
-
-
-console = Console()
-
-
-@click.group()
-def cli():
-    """Manage Crawl4AI Docker server instances
-
-    \b
-    One-command deployment with automatic scaling:
-      • Single container for development (N=1)
-      • Docker Swarm for production with built-in load balancing (N>1)
-      • Docker Compose + Nginx as fallback (N>1)
-
-    \b
-    Examples:
-      cnode start                    # Single container on port 11235
-      cnode start --replicas 3       # Auto-detect Swarm or Compose
-      cnode start -r 5 --port 8080   # 5 replicas on custom port
-      cnode status                   # Check current deployment
-      cnode scale 10                 # Scale to 10 replicas
-      cnode stop                     # Stop and cleanup
-    """
-    pass
-
-
-@cli.command("start")
-@click.option(
-    "--replicas", "-r",
-    type=int,
-    default=1,
-    help="Number of container replicas (default: 1)"
-)
-@click.option(
-    "--mode",
-    type=click.Choice(["auto", "single", "swarm", "compose"]),
-    default="auto",
-    help="Deployment mode (default: auto-detect)"
-)
-@click.option(
-    "--port", "-p",
-    type=int,
-    default=11235,
-    help="External port to expose (default: 11235)"
-)
-@click.option(
-    "--env-file",
-    type=click.Path(exists=True),
-    help="Path to environment file"
-)
-@click.option(
-    "--image",
-    default="unclecode/crawl4ai:latest",
-    help="Docker image to use (default: unclecode/crawl4ai:latest)"
-)
-def start_cmd(replicas: int, mode: str, port: int, env_file: str, image: str):
-    """Start Crawl4AI server with automatic orchestration.
-
-    Deployment modes:
-    - auto: Automatically choose best mode (default)
-    - single: Single container (N=1 only)
-    - swarm: Docker Swarm with built-in load balancing
-    - compose: Docker Compose + Nginx reverse proxy
-
-    The server will:
-    1. Check if Docker is running
-    2. Validate port availability
-    3. Pull image if needed
-    4. Start container(s) with health checks
-    5. Save state for management
-
-    Examples:
-        # Development: single container
-        cnode start
-
-        # Production: 5 replicas with Swarm
-        cnode start --replicas 5
-
-        # Custom configuration
-        cnode start -r 3 --port 8080 --env-file .env.prod
-    """
-    manager = ServerManager()
-
-    console.print(Panel(
-        f"[cyan]Starting Crawl4AI Server[/cyan]\n\n"
-        f"Replicas: [yellow]{replicas}[/yellow]\n"
-        f"Mode: [yellow]{mode}[/yellow]\n"
-        f"Port: [yellow]{port}[/yellow]\n"
-        f"Image: [yellow]{image}[/yellow]",
-        title="Server Start",
-        border_style="cyan"
-    ))
-
-    with console.status("[cyan]Starting server..."):
-        async def _start():
-            return await manager.start(
-                replicas=replicas,
-                mode=mode,
-                port=port,
-                env_file=env_file,
-                image=image
-            )
-        result = anyio.run(_start)
-
-    if result["success"]:
-        console.print(Panel(
-            f"[green]✓ Server started successfully![/green]\n\n"
-            f"Mode: [cyan]{result.get('state_data', {}).get('mode', mode)}[/cyan]\n"
-            f"URL: [bold]http://localhost:{port}[/bold]\n"
-            f"Health: [bold]http://localhost:{port}/health[/bold]\n"
-            f"Monitor: [bold]http://localhost:{port}/monitor[/bold]",
-            title="Server Running",
-            border_style="green"
-        ))
-    else:
-        error_msg = result.get("error", result.get("message", "Unknown error"))
-        console.print(Panel(
-            f"[red]✗ Failed to start server[/red]\n\n"
-            f"{error_msg}",
-            title="Error",
-            border_style="red"
-        ))
-
-        if "already running" in error_msg.lower():
-            console.print("\n[yellow]Hint: Use 'cnode status' to check current deployment[/yellow]")
-            console.print("[yellow]      Use 'cnode stop' to stop existing server[/yellow]")
-
-
-@cli.command("status")
-def status_cmd():
-    """Show current server status and deployment info.
-
-    Displays:
-    - Running state (up/down)
-    - Deployment mode (single/swarm/compose)
-    - Number of replicas
-    - Port mapping
-    - Uptime
-    - Image version
-
-    Example:
-        cnode status
-    """
-    manager = ServerManager()
-
-    async def _status():
-        return await manager.status()
-    result = anyio.run(_status)
-
-    if result["running"]:
-        table = Table(title="Crawl4AI Server Status", border_style="green")
-        table.add_column("Property", style="cyan")
-        table.add_column("Value", style="green")
-
-        table.add_row("Status", "🟢 Running")
-        table.add_row("Mode", result["mode"])
-        table.add_row("Replicas", str(result.get("replicas", 1)))
-        table.add_row("Port", str(result.get("port", 11235)))
-        table.add_row("Image", result.get("image", "unknown"))
-        table.add_row("Uptime", result.get("uptime", "unknown"))
-        table.add_row("Started", result.get("started_at", "unknown"))
-
-        console.print(table)
-        console.print(f"\n[green]✓ Server is healthy[/green]")
-        console.print(f"[dim]Access: http://localhost:{result.get('port', 11235)}[/dim]")
-    else:
-        console.print(Panel(
-            f"[yellow]No server is currently running[/yellow]\n\n"
-            f"Use 'cnode start' to launch a server",
-            title="Server Status",
-            border_style="yellow"
-        ))
-
-
-@cli.command("stop")
-@click.option(
-    "--remove-volumes",
-    is_flag=True,
-    help="Remove associated volumes (WARNING: deletes data)"
-)
-def stop_cmd(remove_volumes: bool):
-    """Stop running Crawl4AI server and cleanup resources.
-
-    This will:
-    1. Stop all running containers/services
-    2. Remove containers
-    3. Optionally remove volumes (--remove-volumes)
-    4. Clean up state files
-
-    WARNING: Use --remove-volumes with caution as it will delete
-    persistent data including Redis databases and logs.
-
-    Examples:
-        # Stop server, keep volumes
-        cnode stop
-
-        # Stop and remove all data
-        cnode stop --remove-volumes
-    """
-    manager = ServerManager()
-
-    # Confirm if removing volumes
-    if remove_volumes:
-        if not Confirm.ask(
-            "[red]⚠️  This will delete all server data including Redis databases. Continue?[/red]"
-        ):
-            console.print("[yellow]Cancelled[/yellow]")
-            return
-
-    with console.status("[cyan]Stopping server..."):
-        async def _stop():
-            return await manager.stop(remove_volumes=remove_volumes)
-        result = anyio.run(_stop)
-
-    if result["success"]:
-        console.print(Panel(
-            f"[green]✓ Server stopped successfully[/green]\n\n"
-            f"{result.get('message', 'All resources cleaned up')}",
-            title="Server Stopped",
-            border_style="green"
-        ))
-    else:
-        console.print(Panel(
-            f"[red]✗ Error stopping server[/red]\n\n"
-            f"{result.get('error', result.get('message', 'Unknown error'))}",
-            title="Error",
-            border_style="red"
-        ))
-
-
-@cli.command("scale")
-@click.argument("replicas", type=int)
-def scale_cmd(replicas: int):
-    """Scale server to specified number of replicas.
-
-    Only works with Swarm or Compose modes. Single container
-    mode cannot be scaled (must stop and restart with --replicas).
-
-    Scaling is live and does not require downtime. The load
-    balancer will automatically distribute traffic to new replicas.
-
-    Examples:
-        # Scale up to 10 replicas
-        cnode scale 10
-
-        # Scale down to 2 replicas
-        cnode scale 2
-
-        # Scale to 1 (minimum)
-        cnode scale 1
-    """
-    if replicas < 1:
-        console.print("[red]Error: Replicas must be at least 1[/red]")
-        return
-
-    manager = ServerManager()
-
-    with console.status(f"[cyan]Scaling to {replicas} replicas..."):
-        async def _scale():
-            return await manager.scale(replicas=replicas)
-        result = anyio.run(_scale)
-
-    if result["success"]:
-        console.print(Panel(
-            f"[green]✓ Scaled successfully[/green]\n\n"
-            f"New replica count: [bold]{replicas}[/bold]\n"
-            f"Mode: [cyan]{result.get('mode')}[/cyan]",
-            title="Scaling Complete",
-            border_style="green"
-        ))
-    else:
-        error_msg = result.get("error", result.get("message", "Unknown error"))
-        console.print(Panel(
-            f"[red]✗ Scaling failed[/red]\n\n"
-            f"{error_msg}",
-            title="Error",
-            border_style="red"
-        ))
-
-        if "single container" in error_msg.lower():
-            console.print("\n[yellow]Hint: For single container mode:[/yellow]")
-            console.print("[yellow]  1. cnode stop[/yellow]")
-            console.print(f"[yellow]  2. cnode start --replicas {replicas}[/yellow]")
-
-
-@cli.command("logs")
-@click.option(
-    "--follow", "-f",
-    is_flag=True,
-    help="Follow log output (like tail -f)"
-)
-@click.option(
-    "--tail",
-    type=int,
-    default=100,
-    help="Number of lines to show (default: 100)"
-)
-def logs_cmd(follow: bool, tail: int):
-    """View server logs.
-
-    Shows logs from running containers/services. Use --follow
-    to stream logs in real-time.
-
-    Examples:
-        # Show last 100 lines
-        cnode logs
-
-        # Show last 500 lines
-        cnode logs --tail 500
-
-        # Follow logs in real-time
-        cnode logs --follow
-
-        # Combine options
-        cnode logs -f --tail 50
-    """
-    manager = ServerManager()
-
-    async def _logs():
-        return await manager.logs(follow=follow, tail=tail)
-    output = anyio.run(_logs)
-    console.print(output)
-
-
-@cli.command("cleanup")
-@click.option(
-    "--force",
-    is_flag=True,
-    help="Force cleanup even if state file doesn't exist"
-)
-def cleanup_cmd(force: bool):
-    """Force cleanup of all Crawl4AI Docker resources.
-
-    Stops and removes all containers, networks, and optionally volumes.
-    Useful when server is stuck or state is corrupted.
-
-    Examples:
-        # Clean up everything
-        cnode cleanup
-
-        # Force cleanup (ignore state file)
-        cnode cleanup --force
-    """
-    manager = ServerManager()
-
-    console.print(Panel(
-        f"[yellow]⚠️  Cleaning up Crawl4AI Docker resources[/yellow]\n\n"
-        f"This will stop and remove:\n"
-        f"- All Crawl4AI containers\n"
-        f"- Nginx load balancer\n"
-        f"- Redis instance\n"
-        f"- Docker networks\n"
-        f"- State files",
-        title="Cleanup",
-        border_style="yellow"
-    ))
-
-    if not force and not Confirm.ask("[yellow]Continue with cleanup?[/yellow]"):
-        console.print("[yellow]Cancelled[/yellow]")
-        return
-
-    with console.status("[cyan]Cleaning up resources..."):
-        async def _cleanup():
-            return await manager.cleanup(force=force)
-        result = anyio.run(_cleanup)
-
-    if result["success"]:
-        console.print(Panel(
-            f"[green]✓ Cleanup completed successfully[/green]\n\n"
-            f"Removed: {result.get('removed', 0)} containers\n"
-            f"{result.get('message', 'All resources cleaned up')}",
-            title="Cleanup Complete",
-            border_style="green"
-        ))
-    else:
-        console.print(Panel(
-            f"[yellow]⚠️  Partial cleanup[/yellow]\n\n"
-            f"{result.get('message', 'Some resources may still exist')}",
-            title="Cleanup Status",
-            border_style="yellow"
-        ))
-
-
-@cli.command("restart")
-@click.option(
-    "--replicas", "-r",
-    type=int,
-    help="New replica count (optional)"
-)
-def restart_cmd(replicas: int):
-    """Restart server (stop then start with same config).
-
-    Preserves existing configuration unless overridden with options.
-    Useful for applying image updates or recovering from errors.
-
-    Examples:
-        # Restart with same configuration
-        cnode restart
-
-        # Restart and change replica count
-        cnode restart --replicas 5
-    """
-    manager = ServerManager()
-
-    # Get current state
-    async def _get_status():
-        return await manager.status()
-    current = anyio.run(_get_status)
-
-    if not current["running"]:
-        console.print("[yellow]No server is running. Use 'cnode start' instead.[/yellow]")
-        return
-
-    # Extract current config
-    current_replicas = current.get("replicas", 1)
-    current_port = current.get("port", 11235)
-    current_image = current.get("image", "unclecode/crawl4ai:latest")
-    current_mode = current.get("mode", "auto")
-
-    # Override with CLI args
-    new_replicas = replicas if replicas is not None else current_replicas
-
-    console.print(Panel(
-        f"[cyan]Restarting Crawl4AI Server[/cyan]\n\n"
-        f"Replicas: [yellow]{current_replicas}[/yellow] → [green]{new_replicas}[/green]\n"
-        f"Port: [yellow]{current_port}[/yellow]\n"
-        f"Mode: [yellow]{current_mode}[/yellow]",
-        title="Server Restart",
-        border_style="cyan"
-    ))
-
-    # Stop current
-    with console.status("[cyan]Stopping current server..."):
-        async def _stop_server():
-            return await manager.stop(remove_volumes=False)
-        stop_result = anyio.run(_stop_server)
-
-    if not stop_result["success"]:
-        console.print(f"[red]Failed to stop server: {stop_result.get('error')}[/red]")
-        return
-
-    # Start new
-    with console.status("[cyan]Starting server..."):
-        async def _start_server():
-            return await manager.start(
-                replicas=new_replicas,
-                mode="auto",
-                port=current_port,
-                image=current_image
-            )
-        start_result = anyio.run(_start_server)
-
-    if start_result["success"]:
-        console.print(Panel(
-            f"[green]✓ Server restarted successfully![/green]\n\n"
-            f"URL: [bold]http://localhost:{current_port}[/bold]",
-            title="Restart Complete",
-            border_style="green"
-        ))
-    else:
-        console.print(Panel(
-            f"[red]✗ Failed to restart server[/red]\n\n"
-            f"{start_result.get('error', 'Unknown error')}",
-            title="Error",
-            border_style="red"
-        ))
-
-
-def main():
-    """Entry point for cnode CLI"""
-    cli()
-
-
-if __name__ == "__main__":
-    main()
-
-# Test comment

deploy/installer/cnode_pkg/requirements.txt

@@ -1,4 +0,0 @@
-click>=8.0.0
-rich>=13.0.0
-anyio>=3.0.0
-pyyaml>=6.0.0

deploy/installer/install-cnode.sh

@@ -1,171 +0,0 @@
-#!/bin/bash
-# Crawl4AI Node Manager (cnode) Remote Installation Script
-# Usage: curl -sSL https://crawl4ai.com/install-cnode.sh | bash
-# Or: wget -qO- https://crawl4ai.com/install-cnode.sh | bash
-
-set -e
-
-# Colors
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-BLUE='\033[0;34m'
-NC='\033[0m'
-
-# Configuration
-INSTALL_DIR="${INSTALL_DIR:-/usr/local/bin}"
-LIB_DIR="${LIB_DIR:-/usr/local/lib/cnode}"
-GITHUB_REPO="unclecode/crawl4ai"
-BRANCH="${CNODE_BRANCH:-main}"
-
-echo -e "${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}"
-echo -e "${GREEN}║   Crawl4AI Node Manager (cnode) Installation Script         ║${NC}"
-echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}\n"
-
-# Check Python
-echo -e "${BLUE}Checking Python installation...${NC}"
-if command -v python3 &> /dev/null; then
-    PYTHON_CMD="python3"
-elif command -v python &> /dev/null; then
-    PYTHON_CMD="python"
-else
-    echo -e "${RED}Error: Python 3.8+ is required but not found${NC}"
-    echo -e "${YELLOW}Install from: https://www.python.org/downloads/${NC}"
-    exit 1
-fi
-
-# Check Python version
-PYTHON_VERSION=$($PYTHON_CMD -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
-echo -e "${GREEN}✓ Found Python $PYTHON_VERSION${NC}"
-
-if [ "$(printf '%s\n' "3.8" "$PYTHON_VERSION" | sort -V | head -n1)" != "3.8" ]; then
-    echo -e "${RED}Error: Python 3.8+ required, found $PYTHON_VERSION${NC}"
-    exit 1
-fi
-
-# Check pip
-if ! $PYTHON_CMD -m pip --version &> /dev/null; then
-    echo -e "${RED}Error: pip is required${NC}"
-    echo -e "${YELLOW}Install pip: $PYTHON_CMD -m ensurepip${NC}"
-    exit 1
-fi
-echo -e "${GREEN}✓ pip is available${NC}"
-
-# Check Docker
-echo -e "\n${BLUE}Checking Docker...${NC}"
-if ! command -v docker &> /dev/null; then
-    echo -e "${YELLOW}⚠️  Docker not found (required for running servers)${NC}"
-    echo -e "${YELLOW}Install from: https://docs.docker.com/get-docker/${NC}\n"
-else
-    echo -e "${GREEN}✓ Docker is installed${NC}"
-fi
-
-# Check permissions
-USE_SUDO=""
-if [ ! -w "$INSTALL_DIR" ] || [ ! -w "/usr/local" ]; then
-    echo -e "\n${YELLOW}⚠️  Root permission required for installation${NC}"
-    USE_SUDO="sudo"
-fi
-
-# Create temp directory
-TMP_DIR="$(mktemp -d)"
-cd "$TMP_DIR"
-
-# Download only cnode_pkg from GitHub using sparse checkout
-echo -e "\n${BLUE}Downloading cnode package from GitHub...${NC}"
-
-if ! command -v git &> /dev/null; then
-    echo -e "${RED}Error: git is required but not found${NC}"
-    echo -e "${YELLOW}Install git and try again${NC}"
-    rm -rf "$TMP_DIR"
-    exit 1
-fi
-
-# Initialize sparse checkout
-git init -q
-git remote add origin "https://github.com/$GITHUB_REPO.git"
-git config core.sparseCheckout true
-
-# Only checkout the cnode_pkg directory
-echo "deploy/installer/cnode_pkg/*" > .git/info/sparse-checkout
-
-# Pull only the needed files
-if ! git pull -q --depth=1 origin "$BRANCH"; then
-    echo -e "${RED}Error: Failed to download package${NC}"
-    rm -rf "$TMP_DIR"
-    exit 1
-fi
-
-if [ ! -d "deploy/installer/cnode_pkg" ]; then
-    echo -e "${RED}Error: Package directory not found${NC}"
-    rm -rf "$TMP_DIR"
-    exit 1
-fi
-
-echo -e "${GREEN}✓ Package downloaded${NC}"
-
-REPO_DIR="."
-
-# Install Python dependencies
-echo -e "\n${BLUE}Installing Python dependencies...${NC}"
-$PYTHON_CMD -m pip install --quiet --user -r "$REPO_DIR/deploy/installer/cnode_pkg/requirements.txt" 2>/dev/null || \
-$PYTHON_CMD -m pip install --quiet --user --break-system-packages -r "$REPO_DIR/deploy/installer/cnode_pkg/requirements.txt" 2>/dev/null || {
-    echo -e "${YELLOW}⚠️  Could not install dependencies with pip${NC}"
-    echo -e "${YELLOW}Trying to continue anyway (dependencies may already be installed)${NC}"
-}
-echo -e "${GREEN}✓ Dependencies check complete${NC}"
-
-# Install cnode package
-echo -e "\n${BLUE}Installing cnode package...${NC}"
-$USE_SUDO mkdir -p "$LIB_DIR"
-$USE_SUDO cp -r "$REPO_DIR/deploy/installer/cnode_pkg" "$LIB_DIR/"
-echo -e "${GREEN}✓ Package installed to $LIB_DIR${NC}"
-
-# Create wrapper script
-echo -e "\n${BLUE}Creating cnode command...${NC}"
-$USE_SUDO tee "$INSTALL_DIR/cnode" > /dev/null << 'EOF'
-#!/usr/bin/env bash
-# Crawl4AI Node Manager (cnode) wrapper
-
-set -e
-
-# Find Python
-if command -v python3 &> /dev/null; then
-    PYTHON_CMD="python3"
-elif command -v python &> /dev/null; then
-    PYTHON_CMD="python"
-else
-    echo "Error: Python 3.8+ required" >&2
-    exit 1
-fi
-
-# Add cnode to Python path and run
-export PYTHONPATH="/usr/local/lib/cnode:$PYTHONPATH"
-exec $PYTHON_CMD -m cnode_pkg.cli "$@"
-EOF
-
-$USE_SUDO chmod +x "$INSTALL_DIR/cnode"
-echo -e "${GREEN}✓ cnode command created${NC}"
-
-# Cleanup
-rm -rf "$TMP_DIR"
-
-echo -e "\n${GREEN}✓ Installation complete${NC}"
-
-# Success message
-echo -e "\n${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}"
-echo -e "${GREEN}║              Installation Complete!                          ║${NC}"
-echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}\n"
-
-echo -e "${BLUE}cnode is now installed and ready!${NC}\n"
-
-echo -e "${YELLOW}Quick Start:${NC}"
-echo -e "  ${GREEN}cnode start${NC}                    # Start single server"
-echo -e "  ${GREEN}cnode start --replicas 5${NC}       # Start 5-replica cluster"
-echo -e "  ${GREEN}cnode status${NC}                   # Check status"
-echo -e "  ${GREEN}cnode logs -f${NC}                  # Follow logs"
-echo -e "  ${GREEN}cnode stop${NC}                     # Stop server"
-
-echo -e "\n${YELLOW}More help:${NC}"
-echo -e "  ${BLUE}cnode --help${NC}"
-echo -e "  ${BLUE}https://github.com/$GITHUB_REPO${NC}\n"

deploy/installer/sync-cnode.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-# Sync cnode source code to installer package
-# Run this before committing changes to cnode
-
-set -e
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-SOURCE_DIR="$SCRIPT_DIR/../docker"
-PKG_DIR="$SCRIPT_DIR/cnode_pkg"
-
-echo "🔄 Syncing cnode source to package..."
-
-# Copy CLI
-echo "  → Copying cnode_cli.py to cli.py"
-cp "$SOURCE_DIR/cnode_cli.py" "$PKG_DIR/cli.py"
-
-# Fix imports
-echo "  → Fixing imports (deploy.docker → cnode_pkg)"
-sed -i '' 's/from deploy\.docker\./from cnode_pkg./g' "$PKG_DIR/cli.py"
-
-# Copy server manager
-echo "  → Copying server_manager.py"
-cp "$SOURCE_DIR/server_manager.py" "$PKG_DIR/server_manager.py"
-
-echo "✅ Sync complete!"
-echo ""
-echo "Files updated:"
-echo "  • deploy/installer/cnode_pkg/cli.py"
-echo "  • deploy/installer/cnode_pkg/server_manager.py"
-echo ""
-echo "Next steps:"
-echo "  1. Test: cd deploy/installer && ./install-cnode.sh"
-echo "  2. Commit both source and package files"

docker-compose.yml

@@ -1,18 +1,43 @@
 version: '3.8'
 
+# Shared configuration for all environments
+x-base-config: &base-config
+  ports:
+    - "11235:11235"  # Gunicorn port
+  env_file:
+    - .llm.env       # API keys (create from .llm.env.example)
+  environment:
+    - OPENAI_API_KEY=${OPENAI_API_KEY:-}
+    - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-}
+    - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
+    - GROQ_API_KEY=${GROQ_API_KEY:-}
+    - TOGETHER_API_KEY=${TOGETHER_API_KEY:-}
+    - MISTRAL_API_KEY=${MISTRAL_API_KEY:-}
+    - GEMINI_API_TOKEN=${GEMINI_API_TOKEN:-}
+    - LLM_PROVIDER=${LLM_PROVIDER:-}  # Optional: Override default provider (e.g., "anthropic/claude-3-opus")
+  volumes:
+    - /dev/shm:/dev/shm  # Chromium performance
+  deploy:
+    resources:
+      limits:
+        memory: 4G
+      reservations:
+        memory: 1G
+  restart: unless-stopped
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:11235/health"]
+    interval: 30s
+    timeout: 10s
+    retries: 3
+    start_period: 40s
+  user: "appuser"
+
 services:
-  redis:
-    image: redis:alpine
-    command: redis-server --appendonly yes
-    volumes:
-      - redis_data:/data
-    networks:
-      - crawl4ai_net
-    restart: unless-stopped
-
   crawl4ai:
+    # 1. Default: Pull multi-platform test image from Docker Hub
+    # 2. Override with local image via: IMAGE=local-test docker compose up
     image: ${IMAGE:-unclecode/crawl4ai:${TAG:-latest}}
-
+    
     # Local build config (used with --build)
     build:
       context: .
@@ -20,58 +45,6 @@ services:
       args:
         INSTALL_TYPE: ${INSTALL_TYPE:-default}
         ENABLE_GPU: ${ENABLE_GPU:-false}
-
+    
-    # No ports exposed - access via nginx only
+    # Inherit shared config
-    env_file:
+    <<: *base-config
-      - .llm.env
-    environment:
-      - OPENAI_API_KEY=${OPENAI_API_KEY:-}
-      - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-}
-      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
-      - GROQ_API_KEY=${GROQ_API_KEY:-}
-      - TOGETHER_API_KEY=${TOGETHER_API_KEY:-}
-      - MISTRAL_API_KEY=${MISTRAL_API_KEY:-}
-      - GEMINI_API_TOKEN=${GEMINI_API_TOKEN:-}
-      - LLM_PROVIDER=${LLM_PROVIDER:-}
-      - REDIS_HOST=redis
-      - REDIS_PORT=6379
-    volumes:
-      - /dev/shm:/dev/shm  # Chromium performance
-    deploy:
-      replicas: 3  # Default to 3 replicas (can override with --scale)
-      resources:
-        limits:
-          memory: 4G
-        reservations:
-          memory: 1G
-    restart: unless-stopped
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:11235/health"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-      start_period: 40s
-    user: "appuser"
-    depends_on:
-      - redis
-    networks:
-      - crawl4ai_net
-
-  nginx:
-    image: nginx:alpine
-    ports:
-      - "11235:80"  # Expose port 11235 to host
-    volumes:
-      - ./crawl4ai/templates/nginx.conf.template:/etc/nginx/nginx.conf:ro
-    depends_on:
-      - crawl4ai
-    networks:
-      - crawl4ai_net
-    restart: unless-stopped
-
-networks:
-  crawl4ai_net:
-    driver: bridge
-
-volumes:
-  redis_data: