#1564 : fix issue with _sig when using proxyConfig

refactor(crawler_pool): enhance signature generation with adapter support and improve error handling
feat(api): update marketplace version and build date in root endpoint response
2025-10-28 14:07:44 +08:00 · 2025-10-28 14:02:43 +08:00 · 2025-10-26 11:35:39 +01:00 · 2025-10-26 11:21:29 +01:00
95 changed files with 213 additions and 19190 deletions
--- a/.githooks/pre-commit
+++ b/.githooks/pre-commit
@@ -1,31 +0,0 @@
 #!/bin/bash
 # Pre-commit hook: Auto-sync cnode files when cnode source is modified
 # Colors
 RED='\033[0;31m'
 GREEN='\033[0;32m'
 YELLOW='\033[1;33m'
 NC='\033[0m'
 # Check if cnode source files are being committed
 CNODE_FILES_CHANGED=$(git diff --cached --name-only | grep -E "deploy/docker/(cnode_cli|server_manager)\.py")
 if [ -n "$CNODE_FILES_CHANGED" ]; then
    echo -e "${YELLOW}🔄 cnode source files modified, auto-syncing to package...${NC}"
    # Run sync script
    if [ -f "deploy/installer/sync-cnode.sh" ]; then
        bash deploy/installer/sync-cnode.sh
        # Stage the synced files
        git add deploy/installer/cnode_pkg/cli.py
        git add deploy/installer/cnode_pkg/server_manager.py
        echo -e "${GREEN}✅ cnode package synced and staged${NC}"
    else
        echo -e "${RED}❌ Error: sync-cnode.sh not found${NC}"
        exit 1
    fi
 fi
 exit 0
--- a/.gitignore
+++ b/.gitignore
@@ -185,8 +185,7 @@ Crawl4AI.egg-info/
 requirements0.txt
 a.txt
-# Ignore shell scripts globally, but allow test scripts
+*.sh
 # *.sh
 .idea
 docs/examples/.chainlit/
 docs/examples/.chainlit/*
@@ -283,15 +282,3 @@ docs/apps/linkdin/debug*/
 docs/apps/linkdin/samples/insights/*
 scripts/
 # Databse files
 *.sqlite3
 *.sqlite3-journal
 *.db-journal
 *.db-wal
 *.db-shm
 *.db
 *.rdb
 *.ldb
 .context/
--- a/crawl4ai/cli.py
+++ b/crawl4ai/cli.py
@@ -2,8 +2,6 @@ import click
 import os
 import sys
 import time
 import subprocess
 import shutil
 import humanize
 from typing import Dict, Any, Optional, List
@@ -627,76 +625,6 @@ def cli():
    pass
 # Register server command group (Docker orchestration)
 # Redirect to standalone 'cnode' CLI
@cli.command("server", context_settings=dict(
    ignore_unknown_options=True,
    allow_extra_args=True,
    allow_interspersed_args=False
 ))
@click.pass_context
 def server_cmd(ctx):
    """Manage Crawl4AI Docker server instances (deprecated - use 'cnode')
    This command has been moved to a standalone CLI called 'cnode'.
    For new installations, use:
        curl -sSL https://crawl4ai.com/deploy.sh | bash
    This redirect allows existing scripts to continue working.
    Available commands: start, stop, status, scale, logs
    Use 'crwl server <command> --help' for command-specific help.
    """
    # Check if cnode is installed
    cnode_path = shutil.which("cnode")
    # Get all the args (subcommand + options)
    args = ctx.args
    if not cnode_path:
        console.print(Panel(
            "[yellow]The 'crwl server' command has been moved to a standalone CLI.[/yellow]\n\n"
            "Please install 'cnode' (Crawl4AI Node Manager):\n"
            "[cyan]curl -sSL https://crawl4ai.com/deploy.sh | bash[/cyan]\n\n"
            "After installation, use:\n"
            "[green]cnode <command>[/green] instead of [dim]crwl server <command>[/dim]\n\n"
            "For backward compatibility, we're using the local version for now.",
            title="Server Command Moved",
            border_style="yellow"
        ))
        # Try to use local version
        try:
            import sys
            # Add deploy/docker to path
            deploy_path = str(Path(__file__).parent.parent / 'deploy' / 'docker')
            if deploy_path not in sys.path:
                sys.path.insert(0, deploy_path)
            from cnode_cli import cli as cnode_cli
            # Forward to cnode with the args
            sys.argv = ['cnode'] + args
            cnode_cli(standalone_mode=False)
            sys.exit(0)
        except SystemExit as e:
            # Normal exit from click
            sys.exit(e.code if hasattr(e, 'code') else 0)
        except Exception as e:
            console.print(f"[red]Error: Could not find cnode or local server CLI: {e}[/red]")
            console.print(f"[dim]Details: {e}[/dim]")
            import traceback
            console.print(f"[dim]{traceback.format_exc()}[/dim]")
            sys.exit(1)
    # cnode is installed - forward everything to it
    try:
        result = subprocess.run([cnode_path] + args, check=False)
        sys.exit(result.returncode)
    except Exception as e:
        console.print(f"[red]Error running cnode: {e}[/red]")
        sys.exit(1)
@cli.group("browser")
 def browser_cmd():
    """Manage browser instances for Crawl4AI
@@ -1534,15 +1462,9 @@ def default(url: str, example: bool, browser_config: str, crawler_config: str, f
 def main():
    import sys
-    # Don't auto-insert 'crawl' if the command is recognized
+    if len(sys.argv) < 2 or sys.argv[1] not in cli.commands:
    if len(sys.argv) >= 2 and sys.argv[1] in cli.commands:
        cli()
    elif len(sys.argv) < 2:
        cli()
    else:
        # Unknown command - insert 'crawl' for backward compat
        sys.argv.insert(1, "crawl")
-        cli()
+    cli()
 if __name__ == "__main__":
    main()
--- a/crawl4ai/server_cli.py
+++ b/crawl4ai/server_cli.py
@@ -1,479 +0,0 @@
 """
 Crawl4AI Server CLI Commands
 Provides `crwl server` command group for Docker orchestration.
 """
 import click
 import anyio
 from rich.console import Console
 from rich.table import Table
 from rich.panel import Panel
 from rich.prompt import Confirm
 from crawl4ai.server_manager import ServerManager
 console = Console()
@click.group("server")
 def server_cmd():
    """Manage Crawl4AI Docker server instances
    One-command deployment with automatic scaling:
    - Single container for development (N=1)
    - Docker Swarm for production with built-in load balancing (N>1)
    - Docker Compose + Nginx as fallback (N>1)
    Examples:
        crwl server start                    # Single container on port 11235
        crwl server start --replicas 3       # Auto-detect Swarm or Compose
        crwl server start -r 5 --port 8080   # 5 replicas on custom port
        crwl server status                   # Check current deployment
        crwl server scale 10                 # Scale to 10 replicas
        crwl server stop                     # Stop and cleanup
    """
    pass
@server_cmd.command("start")
@click.option(
    "--replicas", "-r",
    type=int,
    default=1,
    help="Number of container replicas (default: 1)"
 )
@click.option(
    "--mode",
    type=click.Choice(["auto", "single", "swarm", "compose"]),
    default="auto",
    help="Deployment mode (default: auto-detect)"
 )
@click.option(
    "--port", "-p",
    type=int,
    default=11235,
    help="External port to expose (default: 11235)"
 )
@click.option(
    "--env-file",
    type=click.Path(exists=True),
    help="Path to environment file"
 )
@click.option(
    "--image",
    default="unclecode/crawl4ai:latest",
    help="Docker image to use (default: unclecode/crawl4ai:latest)"
 )
 def start_cmd(replicas: int, mode: str, port: int, env_file: str, image: str):
    """Start Crawl4AI server with automatic orchestration.
    Deployment modes:
    - auto: Automatically choose best mode (default)
    - single: Single container (N=1 only)
    - swarm: Docker Swarm with built-in load balancing
    - compose: Docker Compose + Nginx reverse proxy
    The server will:
    1. Check if Docker is running
    2. Validate port availability
    3. Pull image if needed
    4. Start container(s) with health checks
    5. Save state for management
    Examples:
        # Development: single container
        crwl server start
        # Production: 5 replicas with Swarm
        crwl server start --replicas 5
        # Custom configuration
        crwl server start -r 3 --port 8080 --env-file .env.prod
    """
    manager = ServerManager()
    console.print(Panel(
        f"[cyan]Starting Crawl4AI Server[/cyan]\n\n"
        f"Replicas: [yellow]{replicas}[/yellow]\n"
        f"Mode: [yellow]{mode}[/yellow]\n"
        f"Port: [yellow]{port}[/yellow]\n"
        f"Image: [yellow]{image}[/yellow]",
        title="Server Start",
        border_style="cyan"
    ))
    with console.status("[cyan]Starting server..."):
        async def _start():
            return await manager.start(
                replicas=replicas,
                mode=mode,
                port=port,
                env_file=env_file,
                image=image
            )
        result = anyio.run(_start)
    if result["success"]:
        console.print(Panel(
            f"[green]✓ Server started successfully![/green]\n\n"
            f"Mode: [cyan]{result.get('state_data', {}).get('mode', mode)}[/cyan]\n"
            f"URL: [bold]http://localhost:{port}[/bold]\n"
            f"Health: [bold]http://localhost:{port}/health[/bold]\n"
            f"Monitor: [bold]http://localhost:{port}/monitor[/bold]",
            title="Server Running",
            border_style="green"
        ))
    else:
        error_msg = result.get("error", result.get("message", "Unknown error"))
        console.print(Panel(
            f"[red]✗ Failed to start server[/red]\n\n"
            f"{error_msg}",
            title="Error",
            border_style="red"
        ))
        if "already running" in error_msg.lower():
            console.print("\n[yellow]Hint: Use 'crwl server status' to check current deployment[/yellow]")
            console.print("[yellow]      Use 'crwl server stop' to stop existing server[/yellow]")
@server_cmd.command("status")
 def status_cmd():
    """Show current server status and deployment info.
    Displays:
    - Running state (up/down)
    - Deployment mode (single/swarm/compose)
    - Number of replicas
    - Port mapping
    - Uptime
    - Image version
    Example:
        crwl server status
    """
    manager = ServerManager()
    async def _status():
        return await manager.status()
    result = anyio.run(_status)
    if result["running"]:
        table = Table(title="Crawl4AI Server Status", border_style="green")
        table.add_column("Property", style="cyan")
        table.add_column("Value", style="green")
        table.add_row("Status", "🟢 Running")
        table.add_row("Mode", result["mode"])
        table.add_row("Replicas", str(result.get("replicas", 1)))
        table.add_row("Port", str(result.get("port", 11235)))
        table.add_row("Image", result.get("image", "unknown"))
        table.add_row("Uptime", result.get("uptime", "unknown"))
        table.add_row("Started", result.get("started_at", "unknown"))
        console.print(table)
        console.print(f"\n[green]✓ Server is healthy[/green]")
        console.print(f"[dim]Access: http://localhost:{result.get('port', 11235)}[/dim]")
    else:
        console.print(Panel(
            f"[yellow]No server is currently running[/yellow]\n\n"
            f"Use 'crwl server start' to launch a server",
            title="Server Status",
            border_style="yellow"
        ))
@server_cmd.command("stop")
@click.option(
    "--remove-volumes",
    is_flag=True,
    help="Remove associated volumes (WARNING: deletes data)"
 )
 def stop_cmd(remove_volumes: bool):
    """Stop running Crawl4AI server and cleanup resources.
    This will:
    1. Stop all running containers/services
    2. Remove containers
    3. Optionally remove volumes (--remove-volumes)
    4. Clean up state files
    WARNING: Use --remove-volumes with caution as it will delete
    persistent data including Redis databases and logs.
    Examples:
        # Stop server, keep volumes
        crwl server stop
        # Stop and remove all data
        crwl server stop --remove-volumes
    """
    manager = ServerManager()
    # Confirm if removing volumes
    if remove_volumes:
        if not Confirm.ask(
            "[red]⚠️  This will delete all server data including Redis databases. Continue?[/red]"
        ):
            console.print("[yellow]Cancelled[/yellow]")
            return
    with console.status("[cyan]Stopping server..."):
        async def _stop():
            return await manager.stop(remove_volumes=remove_volumes)
        result = anyio.run(_stop)
    if result["success"]:
        console.print(Panel(
            f"[green]✓ Server stopped successfully[/green]\n\n"
            f"{result.get('message', 'All resources cleaned up')}",
            title="Server Stopped",
            border_style="green"
        ))
    else:
        console.print(Panel(
            f"[red]✗ Error stopping server[/red]\n\n"
            f"{result.get('error', result.get('message', 'Unknown error'))}",
            title="Error",
            border_style="red"
        ))
@server_cmd.command("scale")
@click.argument("replicas", type=int)
 def scale_cmd(replicas: int):
    """Scale server to specified number of replicas.
    Only works with Swarm or Compose modes. Single container
    mode cannot be scaled (must stop and restart with --replicas).
    Scaling is live and does not require downtime. The load
    balancer will automatically distribute traffic to new replicas.
    Examples:
        # Scale up to 10 replicas
        crwl server scale 10
        # Scale down to 2 replicas
        crwl server scale 2
        # Scale to 1 (minimum)
        crwl server scale 1
    """
    if replicas < 1:
        console.print("[red]Error: Replicas must be at least 1[/red]")
        return
    manager = ServerManager()
    with console.status(f"[cyan]Scaling to {replicas} replicas..."):
        async def _scale():
            return await manager.scale(replicas=replicas)
        result = anyio.run(_scale)
    if result["success"]:
        console.print(Panel(
            f"[green]✓ Scaled successfully[/green]\n\n"
            f"New replica count: [bold]{replicas}[/bold]\n"
            f"Mode: [cyan]{result.get('mode')}[/cyan]",
            title="Scaling Complete",
            border_style="green"
        ))
    else:
        error_msg = result.get("error", result.get("message", "Unknown error"))
        console.print(Panel(
            f"[red]✗ Scaling failed[/red]\n\n"
            f"{error_msg}",
            title="Error",
            border_style="red"
        ))
        if "single container" in error_msg.lower():
            console.print("\n[yellow]Hint: For single container mode:[/yellow]")
            console.print("[yellow]  1. crwl server stop[/yellow]")
            console.print(f"[yellow]  2. crwl server start --replicas {replicas}[/yellow]")
@server_cmd.command("logs")
@click.option(
    "--follow", "-f",
    is_flag=True,
    help="Follow log output (like tail -f)"
 )
@click.option(
    "--tail",
    type=int,
    default=100,
    help="Number of lines to show (default: 100)"
 )
 def logs_cmd(follow: bool, tail: int):
    """View server logs.
    Shows logs from running containers/services. Use --follow
    to stream logs in real-time.
    Examples:
        # Show last 100 lines
        crwl server logs
        # Show last 500 lines
        crwl server logs --tail 500
        # Follow logs in real-time
        crwl server logs --follow
        # Combine options
        crwl server logs -f --tail 50
    """
    manager = ServerManager()
    async def _logs():
        return await manager.logs(follow=follow, tail=tail)
    output = anyio.run(_logs)
    console.print(output)
@server_cmd.command("cleanup")
@click.option(
    "--force",
    is_flag=True,
    help="Force cleanup even if state file doesn't exist"
 )
 def cleanup_cmd(force: bool):
    """Force cleanup of all Crawl4AI Docker resources.
    Stops and removes all containers, networks, and optionally volumes.
    Useful when server is stuck or state is corrupted.
    Examples:
        # Clean up everything
        crwl server cleanup
        # Force cleanup (ignore state file)
        crwl server cleanup --force
    """
    manager = ServerManager()
    console.print(Panel(
        f"[yellow]⚠️  Cleaning up Crawl4AI Docker resources[/yellow]\n\n"
        f"This will stop and remove:\n"
        f"- All Crawl4AI containers\n"
        f"- Nginx load balancer\n"
        f"- Redis instance\n"
        f"- Docker networks\n"
        f"- State files",
        title="Cleanup",
        border_style="yellow"
    ))
    if not force and not Confirm.ask("[yellow]Continue with cleanup?[/yellow]"):
        console.print("[yellow]Cancelled[/yellow]")
        return
    with console.status("[cyan]Cleaning up resources..."):
        async def _cleanup():
            return await manager.cleanup(force=force)
        result = anyio.run(_cleanup)
    if result["success"]:
        console.print(Panel(
            f"[green]✓ Cleanup completed successfully[/green]\n\n"
            f"Removed: {result.get('removed', 0)} containers\n"
            f"{result.get('message', 'All resources cleaned up')}",
            title="Cleanup Complete",
            border_style="green"
        ))
    else:
        console.print(Panel(
            f"[yellow]⚠️  Partial cleanup[/yellow]\n\n"
            f"{result.get('message', 'Some resources may still exist')}",
            title="Cleanup Status",
            border_style="yellow"
        ))
@server_cmd.command("restart")
@click.option(
    "--replicas", "-r",
    type=int,
    help="New replica count (optional)"
 )
 def restart_cmd(replicas: int):
    """Restart server (stop then start with same config).
    Preserves existing configuration unless overridden with options.
    Useful for applying image updates or recovering from errors.
    Examples:
        # Restart with same configuration
        crwl server restart
        # Restart and change replica count
        crwl server restart --replicas 5
    """
    manager = ServerManager()
    # Get current state
    async def _get_status():
        return await manager.status()
    current = anyio.run(_get_status)
    if not current["running"]:
        console.print("[yellow]No server is running. Use 'crwl server start' instead.[/yellow]")
        return
    # Extract current config
    current_replicas = current.get("replicas", 1)
    current_port = current.get("port", 11235)
    current_image = current.get("image", "unclecode/crawl4ai:latest")
    current_mode = current.get("mode", "auto")
    # Override with CLI args
    new_replicas = replicas if replicas is not None else current_replicas
    console.print(Panel(
        f"[cyan]Restarting Crawl4AI Server[/cyan]\n\n"
        f"Replicas: [yellow]{current_replicas}[/yellow] → [green]{new_replicas}[/green]\n"
        f"Port: [yellow]{current_port}[/yellow]\n"
        f"Mode: [yellow]{current_mode}[/yellow]",
        title="Server Restart",
        border_style="cyan"
    ))
    # Stop current
    with console.status("[cyan]Stopping current server..."):
        async def _stop_server():
            return await manager.stop(remove_volumes=False)
        stop_result = anyio.run(_stop_server)
    if not stop_result["success"]:
        console.print(f"[red]Failed to stop server: {stop_result.get('error')}[/red]")
        return
    # Start new
    with console.status("[cyan]Starting server..."):
        async def _start_server():
            return await manager.start(
                replicas=new_replicas,
                mode="auto",
                port=current_port,
                image=current_image
            )
        start_result = anyio.run(_start_server)
    if start_result["success"]:
        console.print(Panel(
            f"[green]✓ Server restarted successfully![/green]\n\n"
            f"URL: [bold]http://localhost:{current_port}[/bold]",
            title="Restart Complete",
            border_style="green"
        ))
    else:
        console.print(Panel(
            f"[red]✗ Failed to restart server[/red]\n\n"
            f"{start_result.get('error', 'Unknown error')}",
            title="Error",
            border_style="red"
        ))
--- a/crawl4ai/server_manager.py
+++ b/crawl4ai/server_manager.py
--- a/crawl4ai/templates/docker-compose.template.yml
+++ b/crawl4ai/templates/docker-compose.template.yml
@@ -1,52 +0,0 @@
 version: '3.8'
 services:
  redis:
    image: redis:alpine
    command: redis-server --appendonly yes
    volumes:
      - redis_data:/data
    networks:
      - crawl4ai_net
    restart: unless-stopped
  crawl4ai:
    image: ${IMAGE}
    deploy:
      replicas: ${REPLICAS}
      resources:
        limits:
          memory: 4G
    shm_size: 1g
    environment:
      - REDIS_HOST=redis
      - REDIS_PORT=6379
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:11235/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    depends_on:
      - redis
    networks:
      - crawl4ai_net
  nginx:
    image: nginx:alpine
    ports:
      - "${PORT}:80"
    volumes:
      - ${NGINX_CONF}:/etc/nginx/nginx.conf:ro
    depends_on:
      - crawl4ai
    networks:
      - crawl4ai_net
    restart: unless-stopped
 networks:
  crawl4ai_net:
    driver: bridge
 volumes:
  redis_data:
--- a/crawl4ai/templates/nginx.conf.template
+++ b/crawl4ai/templates/nginx.conf.template
@@ -1,75 +0,0 @@
 events {
    worker_connections 1024;
 }
 http {
    upstream crawl4ai_backend {
        # DNS-based load balancing to Docker Compose service
        # Docker Compose provides DNS resolution for service name
        server crawl4ai:11235 max_fails=3 fail_timeout=30s;
        # Keep connections alive
        keepalive 32;
    }
    # Sticky sessions for monitoring (same IP always goes to same container)
    upstream crawl4ai_monitor {
        ip_hash;  # Sticky sessions based on client IP
        server crawl4ai:11235 max_fails=3 fail_timeout=30s;
        keepalive 32;
    }
    server {
        listen 80;
        server_name _;
        # Increase timeouts for long-running crawl operations
        proxy_connect_timeout 300;
        proxy_send_timeout 300;
        proxy_read_timeout 300;
        send_timeout 300;
        # WebSocket endpoint for real-time monitoring (exact match)
        location = /monitor/ws {
            proxy_pass http://crawl4ai_monitor/monitor/ws;
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            # WebSocket timeouts
            proxy_connect_timeout 7d;
            proxy_send_timeout 7d;
            proxy_read_timeout 7d;
        }
        # Monitor and dashboard with sticky sessions (regex location)
        location ~ ^/(monitor|dashboard) {
            proxy_pass http://crawl4ai_monitor;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
        }
        # HTTP endpoints (load balanced)
        location / {
            proxy_pass http://crawl4ai_backend;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            # Support large request bodies (for batch operations)
            client_max_body_size 10M;
        }
        # Health check endpoint (bypass load balancer)
        location /health {
            proxy_pass http://crawl4ai_backend/health;
            access_log off;
        }
    }
 }
--- a/deploy/docker/AGENT.md
+++ b/deploy/docker/AGENT.md
@@ -1,402 +0,0 @@
 # Crawl4AI DevOps Agent Context
 ## Service Overview
 **Crawl4AI**: Browser-based web crawling service with AI extraction. Docker deployment with horizontal scaling (1-N containers), Redis coordination, Nginx load balancing.
 ## Architecture Quick Reference
 ```
 Client → Nginx:11235 → [crawl4ai-1, crawl4ai-2, ...crawl4ai-N] ← Redis
                              ↓
                         Monitor Dashboard
 ```
 **Components:**
 - **Nginx**: Load balancer (round-robin API, sticky monitoring)
 - **Crawl4AI containers**: FastAPI + Playwright browsers
 - **Redis**: Container discovery (heartbeats 30s), monitoring data aggregation
 - **Monitor**: Real-time dashboard at `/dashboard`
 ## CLI Commands
 ### Start/Stop
 ```bash
 crwl server start [-r N] [--port P] [--mode auto|single|swarm|compose] [--env-file F] [--image I]
 crwl server stop [--remove-volumes]
 crwl server restart [-r N]
 ```
 ### Management
 ```bash
 crwl server status        # Show mode, replicas, port, uptime
 crwl server scale N       # Live scaling (Swarm/Compose only)
 crwl server logs [-f] [--tail N]
 ```
 **Defaults**: replicas=1, port=11235, mode=auto, image=unclecode/crawl4ai:latest
 ## Deployment Modes
 | Replicas | Mode | Load Balancer | Use Case |
 |----------|------|---------------|----------|
 | N=1 | single | None | Dev/testing |
 | N>1 | swarm | Built-in | Production (if `docker swarm init` done) |
 | N>1 | compose | Nginx | Production (fallback) |
 **Mode Detection** (when mode=auto):
 1. If N=1 → single
 2. If N>1 & Swarm active → swarm
 3. If N>1 & Swarm inactive → compose
 ## File Locations
 ```
 ~/.crawl4ai/server/
 ├── state.json              # Current deployment state
 ├── docker-compose.yml      # Generated compose file
 └── nginx.conf              # Generated nginx config
 /app/                       # Inside container
 ├── deploy/docker/server.py
 ├── deploy/docker/monitor.py
 ├── deploy/docker/static/monitor/index.html
 └── crawler_pool.py         # Browser pool (PERMANENT, HOT_POOL, COLD_POOL)
 ```
 ## Monitoring & Troubleshooting
 ### Health Checks
 ```bash
 curl http://localhost:11235/health              # Service health
 curl http://localhost:11235/monitor/containers  # Container discovery
 curl http://localhost:11235/monitor/requests    # Aggregated requests
 ```
 ### Dashboard
 - URL: `http://localhost:11235/dashboard/`
 - Features: Container filtering (All/C-1/C-2/C-3), real-time WebSocket, timeline charts
 - WebSocket: `/monitor/ws` (sticky sessions)
 ### Common Issues
 **No containers showing in dashboard:**
 ```bash
 docker exec <redis-container> redis-cli SMEMBERS monitor:active_containers
 docker exec <redis-container> redis-cli KEYS "monitor:heartbeat:*"
 ```
 Wait 30s for heartbeat registration.
 **Load balancing not working:**
 ```bash
 docker exec <nginx-container> cat /etc/nginx/nginx.conf | grep upstream
 docker logs <nginx-container> | grep error
 ```
 Check Nginx upstream has no `ip_hash` for API endpoints.
 **Redis connection errors:**
 ```bash
 docker logs <crawl4ai-container> | grep -i redis
 docker exec <crawl4ai-container> ping redis
 ```
 Verify REDIS_HOST=redis, REDIS_PORT=6379.
 **Containers not scaling:**
 ```bash
 # Swarm
 docker service ls
 docker service ps crawl4ai
 # Compose
 docker compose -f ~/.crawl4ai/server/docker-compose.yml ps
 docker compose -f ~/.crawl4ai/server/docker-compose.yml up -d --scale crawl4ai=N
 ```
 ### Redis Data Structure
 ```
 monitor:active_containers              # SET: {container_ids}
 monitor:heartbeat:{cid}                # STRING: {id, hostname, last_seen} TTL=60s
 monitor:{cid}:active_requests          # STRING: JSON list, TTL=5min
 monitor:{cid}:completed                # STRING: JSON list, TTL=1h
 monitor:{cid}:janitor                  # STRING: JSON list, TTL=1h
 monitor:{cid}:errors                   # STRING: JSON list, TTL=1h
 monitor:endpoint_stats                 # STRING: JSON aggregate, TTL=24h
 ```
 ## Environment Variables
 ### Required for Multi-LLM
 ```bash
 OPENAI_API_KEY=sk-...
 ANTHROPIC_API_KEY=sk-ant-...
 DEEPSEEK_API_KEY=...
 GROQ_API_KEY=...
 TOGETHER_API_KEY=...
 MISTRAL_API_KEY=...
 GEMINI_API_TOKEN=...
 ```
 ### Redis Configuration (Optional)
 ```bash
 REDIS_HOST=redis                       # Default: redis
 REDIS_PORT=6379                        # Default: 6379
 REDIS_TTL_ACTIVE_REQUESTS=300          # Default: 5min
 REDIS_TTL_COMPLETED_REQUESTS=3600      # Default: 1h
 REDIS_TTL_JANITOR_EVENTS=3600          # Default: 1h
 REDIS_TTL_ERRORS=3600                  # Default: 1h
 REDIS_TTL_ENDPOINT_STATS=86400         # Default: 24h
 REDIS_TTL_HEARTBEAT=60                 # Default: 1min
 ```
 ## API Endpoints
 ### Core API
 - `POST /crawl` - Crawl URL (load-balanced)
 - `POST /batch` - Batch crawl (load-balanced)
 - `GET /health` - Health check (load-balanced)
 ### Monitor API (Aggregated from all containers)
 - `GET /monitor/health` - Local container health
 - `GET /monitor/containers` - All active containers
 - `GET /monitor/requests` - All requests (active + completed)
 - `GET /monitor/browsers` - Browser pool status (local only)
 - `GET /monitor/logs/janitor` - Janitor cleanup events
 - `GET /monitor/logs/errors` - Error logs
 - `GET /monitor/endpoints/stats` - Endpoint analytics
 - `WS /monitor/ws` - Real-time updates (aggregated)
 ### Control Actions
 - `POST /monitor/actions/cleanup` - Force browser cleanup
 - `POST /monitor/actions/kill_browser` - Kill specific browser
 - `POST /monitor/actions/restart_browser` - Restart browser
 - `POST /monitor/stats/reset` - Reset endpoint counters
 ## Docker Commands Reference
 ### Inspection
 ```bash
 # List containers
 docker ps --filter "name=crawl4ai"
 # Container logs
 docker logs <container-id> -f --tail 100
 # Redis CLI
 docker exec -it <redis-container> redis-cli
 KEYS monitor:*
 SMEMBERS monitor:active_containers
 GET monitor:<cid>:completed
 TTL monitor:heartbeat:<cid>
 # Nginx config
 docker exec <nginx-container> cat /etc/nginx/nginx.conf
 # Container stats
 docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"
 ```
 ### Compose Operations
 ```bash
 # Scale
 docker compose -f ~/.crawl4ai/server/docker-compose.yml up -d --scale crawl4ai=5
 # Restart service
 docker compose -f ~/.crawl4ai/server/docker-compose.yml restart crawl4ai
 # View services
 docker compose -f ~/.crawl4ai/server/docker-compose.yml ps
 ```
 ### Swarm Operations
 ```bash
 # Initialize Swarm
 docker swarm init
 # Scale service
 docker service scale crawl4ai=5
 # Service info
 docker service ls
 docker service ps crawl4ai --no-trunc
 # Service logs
 docker service logs crawl4ai --tail 100 -f
 ```
 ## Performance & Scaling
 ### Resource Recommendations
 | Containers | Memory/Container | Total Memory | Use Case |
 |------------|-----------------|--------------|----------|
 | 1 | 4GB | 4GB | Development |
 | 3 | 4GB | 12GB | Small prod |
 | 5 | 4GB | 20GB | Medium prod |
 | 10 | 4GB | 40GB | Large prod |
 **Expected Throughput**: ~10 req/min per container (depends on crawl complexity)
 ### Scaling Guidelines
 - **Horizontal**: Add replicas (`crwl server scale N`)
 - **Vertical**: Adjust `--memory 8G --cpus 4` in kwargs
 - **Browser Pool**: Permanent (1) + Hot pool (adaptive) + Cold pool (cleanup by janitor)
 ### Redis Memory Usage
 - **Per container**: ~110KB (requests + events + errors + heartbeat)
 - **10 containers**: ~1.1MB
 - **Recommendation**: 256MB Redis is sufficient for <100 containers
 ## Security Notes
 ### Input Validation
 All CLI inputs validated:
 - Image name: alphanumeric + `.-/:_@` only, max 256 chars
 - Port: 1-65535
 - Replicas: 1-100
 - Env file: must exist and be readable
 - Container IDs: alphanumeric + `-_` only (prevents Redis injection)
 ### Network Security
 - Nginx forwards to internal `crawl4ai` service (Docker network)
 - Monitor endpoints have NO authentication (add MONITOR_TOKEN env for security)
 - Redis is internal-only (no external port)
 ### Recommended Production Setup
 ```bash
 # Add authentication
 export MONITOR_TOKEN="your-secret-token"
 # Use Redis password
 redis:
  command: redis-server --requirepass ${REDIS_PASSWORD}
 # Enable rate limiting in Nginx
 limit_req_zone $binary_remote_addr zone=api:10m rate=10r/s;
 ```
 ## Common User Scenarios
 ### Scenario 1: Fresh Deployment
 ```bash
 crwl server start --replicas 3 --env-file .env
 # Wait for health check, then access http://localhost:11235/health
 ```
 ### Scenario 2: Scaling Under Load
 ```bash
 crwl server scale 10
 # Live scaling, no downtime
 ```
 ### Scenario 3: Debugging Slow Requests
 ```bash
 # Check dashboard
 open http://localhost:11235/dashboard/
 # Check container logs
 docker logs <slowest-container-id> --tail 100
 # Check browser pool
 curl http://localhost:11235/monitor/browsers | jq
 ```
 ### Scenario 4: Redis Connection Issues
 ```bash
 # Check Redis connectivity
 docker exec <crawl4ai-container> nc -zv redis 6379
 # Check Redis logs
 docker logs <redis-container>
 # Restart containers (triggers reconnect with retry logic)
 crwl server restart
 ```
 ### Scenario 5: Container Not Appearing in Dashboard
 ```bash
 # Wait 30s for heartbeat
 sleep 30
 # Check Redis
 docker exec <redis-container> redis-cli SMEMBERS monitor:active_containers
 # Check container logs for heartbeat errors
 docker logs <missing-container> | grep -i heartbeat
 ```
 ## Code Context for Advanced Debugging
 ### Key Classes
 - `MonitorStats` (monitor.py): Tracks stats, Redis persistence, heartbeat worker
 - `ServerManager` (server_manager.py): CLI orchestration, mode detection
 - Browser pool globals: `PERMANENT`, `HOT_POOL`, `COLD_POOL`, `LOCK` (crawler_pool.py)
 ### Critical Timeouts
 - Browser pool lock: 2s timeout (prevents deadlock)
 - WebSocket connection: 5s timeout
 - Health check: 30-60s timeout
 - Heartbeat interval: 30s, TTL: 60s
 - Redis retry: 3 attempts, backoff: 0.5s/1s/2s
 - Circuit breaker: 5 failures → 5min backoff
 ### State Transitions
 ```
 NOT_RUNNING → STARTING → HEALTHY → RUNNING
                ↓           ↓
            FAILED      UNHEALTHY → STOPPED
 ```
 State file: `~/.crawl4ai/server/state.json` (atomic writes, fcntl locking)
 ## Quick Diagnostic Commands
 ```bash
 # Full system check
 crwl server status
 docker ps
 curl http://localhost:11235/health
 curl http://localhost:11235/monitor/containers | jq
 # Redis check
 docker exec <redis-container> redis-cli PING
 docker exec <redis-container> redis-cli INFO stats
 # Network check
 docker network ls
 docker network inspect <network-name>
 # Logs check
 docker logs <nginx-container> --tail 50
 docker logs <redis-container> --tail 50
 docker compose -f ~/.crawl4ai/server/docker-compose.yml logs --tail 100
 ```
 ## Agent Decision Tree
 **User reports slow crawling:**
 1. Check dashboard for active requests stuck → kill browser if >5min
 2. Check browser pool status → cleanup if hot/cold pool >10
 3. Check container CPU/memory → scale up if >80%
 4. Check Redis latency → restart Redis if >100ms
 **User reports missing containers:**
 1. Wait 30s for heartbeat
 2. Check `docker ps` vs dashboard count
 3. Check Redis SMEMBERS monitor:active_containers
 4. Check container logs for Redis connection errors
 5. Verify REDIS_HOST/PORT env vars
 **User reports 502/503 errors:**
 1. Check Nginx logs for upstream errors
 2. Check container health: `curl http://localhost:11235/health`
 3. Check if all containers are healthy: `docker ps`
 4. Restart Nginx: `docker restart <nginx-container>`
 **User wants to update image:**
 1. `crwl server stop`
 2. `docker pull unclecode/crawl4ai:latest`
 3. `crwl server start --replicas <previous-count>`
 ---
 **Version**: Crawl4AI v0.7.4+
 **Last Updated**: 2025-01-20
 **AI Agent Note**: All commands, file paths, and Redis keys verified against codebase. Use exact syntax shown. For user-facing responses, translate technical details to plain language.
--- a/deploy/docker/ARCHITECTURE.md
+++ b/deploy/docker/ARCHITECTURE.md
@@ -1,822 +0,0 @@
 # Crawl4AI Docker Architecture - AI Context Map
 **Purpose:** Dense technical reference for AI agents to understand complete system architecture.
 **Format:** Symbolic, compressed, high-information-density documentation.
 ---
 ## System Overview
 ```
 ┌─────────────────────────────────────────────────────────────┐
 │ CRAWL4AI DOCKER ORCHESTRATION SYSTEM                        │
 ├─────────────────────────────────────────────────────────────┤
 │ Modes: Single (N=1) | Swarm (N>1) | Compose+Nginx (N>1)     │
 │ Entry: cnode CLI → deploy/docker/cnode_cli.py               │
 │ Core: deploy/docker/server_manager.py                       │
 │ Server: deploy/docker/server.py (FastAPI)                   │
 │ API: deploy/docker/api.py (crawl endpoints)                 │
 │ Monitor: deploy/docker/monitor.py + monitor_routes.py       │
 └─────────────────────────────────────────────────────────────┘
 ```
 ---
 ## Directory Structure & File Map
 ```
 deploy/
 ├── docker/                          # Server runtime & orchestration
 │   ├── server.py                    # FastAPI app entry [CRITICAL]
 │   ├── api.py                       # /crawl, /screenshot, /pdf endpoints
 │   ├── server_manager.py            # Docker orchestration logic [CORE]
 │   ├── cnode_cli.py                 # CLI interface (Click-based)
 │   ├── monitor.py                   # Real-time metrics collector
 │   ├── monitor_routes.py            # /monitor dashboard routes
 │   ├── crawler_pool.py              # Browser pool management
 │   ├── hook_manager.py              # Pre/post crawl hooks
 │   ├── job.py                       # Job queue schema
 │   ├── utils.py                     # Helpers (port check, health)
 │   ├── auth.py                      # API key authentication
 │   ├── schemas.py                   # Pydantic models
 │   ├── mcp_bridge.py                # MCP protocol bridge
 │   ├── supervisord.conf             # Process manager config
 │   ├── config.yml                   # Server config template
 │   ├── requirements.txt             # Python deps
 │   ├── static/                      # Web assets
 │   │   ├── monitor/                 # Dashboard UI
 │   │   └── playground/              # API playground
 │   └── tests/                       # Test suite
 │
 └── installer/                       # User-facing installation
    ├── cnode_pkg/                   # Standalone package
    │   ├── cli.py                   # Copy of cnode_cli.py
    │   ├── server_manager.py        # Copy of server_manager.py
    │   └── requirements.txt         # click, rich, anyio, pyyaml
    ├── install-cnode.sh             # Remote installer (git sparse-checkout)
    ├── sync-cnode.sh                # Dev tool (source→pkg sync)
    ├── USER_GUIDE.md                # Human-readable guide
    ├── README.md                    # Developer documentation
    └── QUICKSTART.md                # Cheat sheet
 ```
 ---
 ## Core Components Deep Dive
 ### 1. `server_manager.py` - Orchestration Engine
 **Role:** Manages Docker container lifecycle, auto-detects deployment mode.
 **Key Classes:**
 - `ServerManager` - Main orchestrator
  - `start(replicas, mode, port, env_file, image)` → Deploy server
  - `stop(remove_volumes)` → Teardown
  - `status()` → Health check
  - `scale(replicas)` → Live scaling
  - `logs(follow, tail)` → Stream logs
  - `cleanup(force)` → Emergency cleanup
 **State Management:**
 - File: `~/.crawl4ai/server_state.yml`
 - Schema: `{mode, replicas, port, image, started_at, containers[]}`
 - Atomic writes with lock file
 **Deployment Modes:**
 ```python
 if replicas == 1:
    mode = "single"  # docker run
 elif swarm_available():
    mode = "swarm"   # docker stack deploy
 else:
    mode = "compose" # docker-compose + nginx
 ```
 **Container Naming:**
 - Single: `crawl4ai-server`
 - Swarm: `crawl4ai-stack_crawl4ai`
 - Compose: `crawl4ai-server-{1..N}`, `crawl4ai-nginx`
 **Networks:**
 - `crawl4ai-network` (bridge mode for all)
 **Volumes:**
 - `crawl4ai-redis-data` - Persistent queue
 - `crawl4ai-profiles` - Browser profiles
 **Health Checks:**
 - Endpoint: `http://localhost:{port}/health`
 - Timeout: 30s startup
 - Retry: 3 attempts
 ---
 ### 2. `server.py` - FastAPI Application
 **Role:** HTTP server exposing crawl API + monitoring.
 **Startup Flow:**
 ```python
 app = FastAPI()
@app.on_event("startup")
 async def startup():
    init_crawler_pool()      # Pre-warm browsers
    init_redis_connection()  # Job queue
    start_monitor_collector() # Metrics
 ```
 **Key Endpoints:**
 ```
 POST /crawl          → api.py:crawl_endpoint()
 POST /crawl/stream   → api.py:crawl_stream_endpoint()
 POST /screenshot     → api.py:screenshot_endpoint()
 POST /pdf            → api.py:pdf_endpoint()
 GET  /health         → server.py:health_check()
 GET  /monitor        → monitor_routes.py:dashboard()
 WS   /monitor/ws     → monitor_routes.py:websocket_endpoint()
 GET  /playground     → static/playground/index.html
 ```
 **Process Manager:**
 - Uses `supervisord` to manage:
  - FastAPI server (port 11235)
  - Redis (port 6379)
  - Background workers
 **Environment:**
 ```bash
 CRAWL4AI_PORT=11235
 REDIS_URL=redis://localhost:6379
 MAX_CONCURRENT_CRAWLS=5
 BROWSER_POOL_SIZE=3
 ```
 ---
 ### 3. `api.py` - Crawl Endpoints
 **Main Endpoint:** `POST /crawl`
 **Request Schema:**
 ```json
 {
  "urls": ["https://example.com"],
  "priority": 10,
  "browser_config": {
    "type": "BrowserConfig",
    "params": {"headless": true, "viewport_width": 1920}
  },
  "crawler_config": {
    "type": "CrawlerRunConfig",
    "params": {"cache_mode": "bypass", "extraction_strategy": {...}}
  }
 }
 ```
 **Processing Flow:**
 ```
 1. Validate request (Pydantic)
 2. Queue job → Redis
 3. Get browser from pool → crawler_pool.py
 4. Execute crawl → AsyncWebCrawler
 5. Apply hooks → hook_manager.py
 6. Return result (JSON)
 7. Release browser to pool
 ```
 **Memory Management:**
 - Browser pool: Max 3 instances
 - LRU eviction when pool full
 - Explicit cleanup: `browser.close()` in finally block
 - Redis TTL: 1 hour for completed jobs
 **Error Handling:**
 ```python
 try:
    result = await crawler.arun(url, config)
 except PlaywrightError as e:
    # Browser crash - release & recreate
    await pool.invalidate(browser_id)
 except TimeoutError as e:
    # Timeout - kill & retry
    await crawler.kill()
 except Exception as e:
    # Unknown - log & fail gracefully
    logger.error(f"Crawl failed: {e}")
 ```
 ---
 ### 4. `crawler_pool.py` - Browser Pool Manager
 **Role:** Manage persistent browser instances to avoid startup overhead.
 **Class:** `CrawlerPool`
 - `get_crawler()` → Lease browser (async with context manager)
 - `release_crawler(id)` → Return to pool
 - `warm_up(count)` → Pre-launch browsers
 - `cleanup()` → Close all browsers
 **Pool Strategy:**
 ```python
 pool = {
    "browser_1": {"crawler": AsyncWebCrawler(), "in_use": False},
    "browser_2": {"crawler": AsyncWebCrawler(), "in_use": False},
    "browser_3": {"crawler": AsyncWebCrawler(), "in_use": False},
 }
 async with pool.get_crawler() as crawler:
    result = await crawler.arun(url)
    # Auto-released on context exit
 ```
 **Anti-Leak Mechanisms:**
 1. Context managers enforce cleanup
 2. Watchdog thread kills stale browsers (>10min idle)
 3. Max lifetime: 1 hour per browser
 4. Force GC after browser close
 ---
 ### 5. `monitor.py` + `monitor_routes.py` - Real-time Dashboard
 **Architecture:**
 ```
 [Browser] <--WebSocket--> [monitor_routes.py] <--Events--> [monitor.py]
                              ↓
                          [Redis Pub/Sub]
                              ↓
                       [Metrics Collector]
 ```
 **Metrics Collected:**
 - Requests/sec (sliding window)
 - Active crawls (real-time count)
 - Response times (p50, p95, p99)
 - Error rate (5min rolling)
 - Memory usage (RSS, heap)
 - Browser pool utilization
 **WebSocket Protocol:**
 ```json
 // Server → Client
 {
  "type": "metrics",
  "data": {
    "rps": 45.3,
    "active_crawls": 12,
    "p95_latency": 1234,
    "error_rate": 0.02
  }
 }
 // Client → Server
 {
  "type": "subscribe",
  "channels": ["metrics", "logs"]
 }
 ```
 **Dashboard Route:** `/monitor`
 - Real-time graphs (Chart.js)
 - Request log stream
 - Container health status
 - Resource utilization
 ---
 ### 6. `cnode_cli.py` - CLI Interface
 **Framework:** Click (Python CLI framework)
 **Command Structure:**
 ```
 cnode
 ├── start [--replicas N] [--port P] [--mode M] [--image I]
 ├── stop [--remove-volumes]
 ├── status
 ├── scale N
 ├── logs [--follow] [--tail N]
 ├── restart [--replicas N]
 └── cleanup [--force]
 ```
 **Execution Flow:**
 ```python
@cli.command("start")
 def start_cmd(replicas, mode, port, env_file, image):
    manager = ServerManager()
    result = anyio.run(manager.start(...))  # Async bridge
    if result["success"]:
        console.print(success_panel)
 ```
 **User Feedback:**
 - Rich library for colors/tables
 - Progress spinners during operations
 - Error messages with hints
 - Status tables with health indicators
 **State Persistence:**
 - Saves deployment config to `~/.crawl4ai/server_state.yml`
 - Enables stateless commands (status, scale, restart)
 ---
 ### 7. Docker Orchestration Details
 **Single Container Mode (N=1):**
 ```bash
 docker run -d \
  --name crawl4ai-server \
  --network crawl4ai-network \
  -p 11235:11235 \
  -v crawl4ai-redis-data:/data \
  unclecode/crawl4ai:latest
 ```
 **Docker Swarm Mode (N>1, Swarm available):**
 ```yaml
 # docker-compose.swarm.yml
 version: '3.8'
 services:
  crawl4ai:
    image: unclecode/crawl4ai:latest
    deploy:
      replicas: 5
      update_config:
        parallelism: 2
        delay: 10s
      restart_policy:
        condition: on-failure
    ports:
      - "11235:11235"
    networks:
      - crawl4ai-network
 ```
 Deploy: `docker stack deploy -c docker-compose.swarm.yml crawl4ai-stack`
 **Docker Compose + Nginx Mode (N>1, fallback):**
 ```yaml
 # docker-compose.yml
 services:
  crawl4ai-1:
    image: unclecode/crawl4ai:latest
    networks: [crawl4ai-network]
  crawl4ai-2:
    image: unclecode/crawl4ai:latest
    networks: [crawl4ai-network]
  nginx:
    image: nginx:alpine
    ports: ["11235:80"]
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf
    networks: [crawl4ai-network]
 ```
 Nginx config (round-robin load balancing):
 ```nginx
 upstream crawl4ai_backend {
    server crawl4ai-1:11235;
    server crawl4ai-2:11235;
    server crawl4ai-3:11235;
 }
 server {
    listen 80;
    location / {
        proxy_pass http://crawl4ai_backend;
        proxy_set_header Host $host;
    }
 }
 ```
 ---
 ## Memory Leak Prevention Strategy
 ### Problem Areas & Solutions
 **1. Browser Instances**
 ```python
 # ❌ BAD - Leak risk
 crawler = AsyncWebCrawler()
 result = await crawler.arun(url)
 # Browser never closed!
 # ✅ GOOD - Guaranteed cleanup
 async with AsyncWebCrawler() as crawler:
    result = await crawler.arun(url)
    # Auto-closed on exit
 ```
 **2. WebSocket Connections**
 ```python
 # monitor_routes.py
 active_connections = set()
@app.websocket("/monitor/ws")
 async def websocket_endpoint(websocket):
    await websocket.accept()
    active_connections.add(websocket)
    try:
        while True:
            await websocket.send_json(get_metrics())
    finally:
        active_connections.remove(websocket)  # Critical!
 ```
 **3. Redis Connections**
 ```python
 # Use connection pooling
 redis_pool = aioredis.ConnectionPool(
    host="localhost",
    port=6379,
    max_connections=10,
    decode_responses=True
 )
 # Reuse connections
 async def get_job(job_id):
    async with redis_pool.get_connection() as conn:
        data = await conn.get(f"job:{job_id}")
    # Connection auto-returned to pool
 ```
 **4. Async Task Cleanup**
 ```python
 # Track background tasks
 background_tasks = set()
 async def crawl_task(url):
    try:
        result = await crawl(url)
    finally:
        background_tasks.discard(asyncio.current_task())
 # On shutdown
 async def shutdown():
    tasks = list(background_tasks)
    for task in tasks:
        task.cancel()
    await asyncio.gather(*tasks, return_exceptions=True)
 ```
 **5. File Descriptor Leaks**
 ```python
 # Use context managers for files
 async def save_screenshot(url):
    async with aiofiles.open(f"{job_id}.png", "wb") as f:
        await f.write(screenshot_bytes)
    # File auto-closed
 ```
 ---
 ## Installation & Distribution
 ### User Installation Flow
 **Script:** `deploy/installer/install-cnode.sh`
 **Steps:**
 1. Check Python 3.8+ exists
 2. Check pip available
 3. Check Docker installed (warn if missing)
 4. Create temp dir: `mktemp -d`
 5. Git sparse-checkout:
   ```bash
   git init
   git remote add origin https://github.com/unclecode/crawl4ai.git
   git config core.sparseCheckout true
   echo "deploy/installer/cnode_pkg/*" > .git/info/sparse-checkout
   git pull --depth=1 origin main
   ```
 6. Install deps: `pip install click rich anyio pyyaml`
 7. Copy package: `cnode_pkg/ → /usr/local/lib/cnode/`
 8. Create wrapper: `/usr/local/bin/cnode`
   ```bash
   #!/usr/bin/env bash
   export PYTHONPATH="/usr/local/lib/cnode:$PYTHONPATH"
   exec python3 -m cnode_pkg.cli "$@"
   ```
 9. Cleanup temp dir
 **Result:**
 - Binary-like experience (fast startup: ~0.1s)
 - No need for PyInstaller (49x faster)
 - Platform-independent (any OS with Python)
 ---
 ## Development Workflow
 ### Source Code Sync (Auto)
 **Git Hook:** `.githooks/pre-commit`
 **Trigger:** When committing `deploy/docker/cnode_cli.py` or `server_manager.py`
 **Action:**
 ```bash
 1. Diff source vs package
 2. If different:
   - Run sync-cnode.sh
   - Copy cnode_cli.py → cnode_pkg/cli.py
   - Fix imports: s/deploy.docker/cnode_pkg/g
   - Copy server_manager.py → cnode_pkg/
   - Stage synced files
 3. Continue commit
 ```
 **Setup:** `./setup-hooks.sh` (configures `git config core.hooksPath .githooks`)
 **Smart Behavior:**
 - Silent when no sync needed
 - Only syncs if content differs
 - Minimal output: `✓ cnode synced`
 ---
 ## API Request/Response Flow
 ### Example: POST /crawl
 **Request:**
 ```bash
 curl -X POST http://localhost:11235/crawl \
  -H "Content-Type: application/json" \
  -d '{
    "urls": ["https://example.com"],
    "browser_config": {
      "type": "BrowserConfig",
      "params": {"headless": true}
    },
    "crawler_config": {
      "type": "CrawlerRunConfig",
      "params": {"cache_mode": "bypass"}
    }
  }'
 ```
 **Processing:**
 ```
 1. FastAPI receives request → api.py:crawl_endpoint()
 2. Validate schema → Pydantic models in schemas.py
 3. Create job → job.py:Job(id=uuid4(), urls=[...])
 4. Queue to Redis → LPUSH crawl_queue {job_json}
 5. Get browser from pool → crawler_pool.py:get_crawler()
 6. Execute crawl:
   a. Launch page → browser.new_page()
   b. Navigate → page.goto(url)
   c. Extract → extraction_strategy.extract()
   d. Generate markdown → markdown_generator.generate()
 7. Store result → Redis SETEX result:{job_id} 3600 {result_json}
 8. Release browser → pool.release(browser_id)
 9. Return response:
   {
     "success": true,
     "result": {
       "url": "https://example.com",
       "markdown": "# Example Domain...",
       "metadata": {"title": "Example Domain"},
       "extracted_content": {...}
     }
   }
 ```
 **Error Cases:**
 - 400: Invalid request schema
 - 429: Rate limit exceeded
 - 500: Internal error (browser crash, timeout)
 - 503: Service unavailable (all browsers busy)
 ---
 ## Scaling Behavior
 ### Scale-Up (1 → 10 replicas)
 **Command:** `cnode scale 10`
 **Swarm Mode:**
 ```bash
 docker service scale crawl4ai-stack_crawl4ai=10
 # Docker handles:
 # - Container creation
 # - Network attachment
 # - Load balancer update
 # - Rolling deployment
 ```
 **Compose Mode:**
 ```bash
 # Update docker-compose.yml
 # Change replica count in all service definitions
 docker-compose up -d --scale crawl4ai=10
 # Regenerate nginx.conf with 10 upstreams
 docker exec nginx nginx -s reload
 ```
 **Load Distribution:**
 - Swarm: Built-in ingress network (VIP-based round-robin)
 - Compose: Nginx upstream (round-robin, can configure least_conn)
 **Zero-Downtime:**
 - Swarm: Yes (rolling update, parallelism=2)
 - Compose: Partial (nginx reload is graceful, but brief spike)
 ---
 ## Configuration Files
 ### `config.yml` - Server Configuration
 ```yaml
 server:
  port: 11235
  host: "0.0.0.0"
  workers: 4
 crawler:
  max_concurrent: 5
  timeout: 30
  retries: 3
 browser:
  pool_size: 3
  headless: true
  args:
    - "--no-sandbox"
    - "--disable-dev-shm-usage"
 redis:
  host: "localhost"
  port: 6379
  db: 0
 monitoring:
  enabled: true
  metrics_interval: 5  # seconds
 ```
 ### `supervisord.conf` - Process Management
 ```ini
 [supervisord]
 nodaemon=true
 [program:redis]
 command=redis-server --port 6379
 autorestart=true
 [program:fastapi]
 command=uvicorn server:app --host 0.0.0.0 --port 11235
 autorestart=true
 stdout_logfile=/var/log/crawl4ai/api.log
 [program:monitor]
 command=python monitor.py
 autorestart=true
 ```
 ---
 ## Testing & Quality
 ### Test Structure
 ```
 deploy/docker/tests/
 ├── cli/                    # CLI command tests
 │   └── test_commands.py    # start, stop, scale, status
 ├── monitor/                # Dashboard tests
 │   └── test_websocket.py   # WS connection, metrics
 └── codebase_test/          # Integration tests
    └── test_api.py         # End-to-end crawl tests
 ```
 ### Key Test Cases
 **CLI Tests:**
 - `test_start_single()` - Starts 1 replica
 - `test_start_cluster()` - Starts N replicas
 - `test_scale_up()` - Scales 1→5
 - `test_scale_down()` - Scales 5→2
 - `test_status()` - Reports correct state
 - `test_logs()` - Streams logs
 **API Tests:**
 - `test_crawl_success()` - Basic crawl works
 - `test_crawl_timeout()` - Handles slow sites
 - `test_concurrent_crawls()` - Parallel requests
 - `test_browser_pool()` - Reuses browsers
 - `test_memory_cleanup()` - No leaks after 100 crawls
 **Monitor Tests:**
 - `test_websocket_connect()` - WS handshake
 - `test_metrics_stream()` - Receives updates
 - `test_multiple_clients()` - Handles N connections
 ---
 ## Critical File Cross-Reference
 | Component | Primary File | Dependencies |
 |-----------|--------------|--------------|
 | **CLI Entry** | `cnode_cli.py:482` | `server_manager.py`, `click`, `rich` |
 | **Orchestrator** | `server_manager.py:45` | `docker`, `yaml`, `anyio` |
 | **API Server** | `server.py:120` | `api.py`, `monitor_routes.py` |
 | **Crawl Logic** | `api.py:78` | `crawler_pool.py`, `AsyncWebCrawler` |
 | **Browser Pool** | `crawler_pool.py:23` | `AsyncWebCrawler`, `asyncio` |
 | **Monitoring** | `monitor.py:156` | `redis`, `psutil` |
 | **Dashboard** | `monitor_routes.py:89` | `monitor.py`, `websockets` |
 | **Hooks** | `hook_manager.py:12` | `api.py`, custom user hooks |
 **Startup Chain:**
 ```
 cnode start
  └→ cnode_cli.py:start_cmd()
      └→ server_manager.py:start()
          └→ docker run/stack/compose
              └→ supervisord
                  ├→ redis-server
                  ├→ server.py
                  │   └→ api.py (routes)
                  │   └→ crawler_pool.py (init)
                  └→ monitor.py (collector)
 ```
 ---
 ## Symbolic Notation Summary
 ```
 ⊕ Addition/Creation      ⊖ Removal/Cleanup
 ⊗ Multiplication/Scale   ⊘ Division/Split
 → Flow/Dependency        ← Reverse flow
 ⇄ Bidirectional          ⇵ Up/Down scale
 ✓ Success/Complete       ✗ Failure/Error
 ⚠ Warning                ⚡ Performance critical
 🔒 Lock/Exclusive        🔓 Unlock/Shared
 📊 Metrics               📝 Logs
 🌐 Network               💾 Storage
 🧠 Memory                🐳 Docker
 ```
 **State Machine:**
 ```
 [STOPPED] →start→ [STARTING] →ready→ [RUNNING]
 [RUNNING] →scale→ [SCALING] →done→ [RUNNING]
 [RUNNING] →stop→ [STOPPING] →done→ [STOPPED]
 [ANY] →error→ [ERROR] →cleanup→ [STOPPED]
 ```
 ---
 ## Quick Reference Commands
 ```bash
 # Install
 curl -sSL https://raw.githubusercontent.com/unclecode/crawl4ai/main/deploy/installer/install-cnode.sh | bash
 # Deploy
 cnode start --replicas 5 --port 11235
 # Monitor
 open http://localhost:11235/monitor
 # Scale
 cnode scale 10
 # Test
 curl -X POST http://localhost:11235/crawl -H "Content-Type: application/json" -d '{"urls":["https://example.com"]}'
 # Logs
 cnode logs -f
 # Teardown
 cnode stop --remove-volumes
 ```
 ---
 ## Future AI Agent Instructions
 **When debugging:** Read this file first, then check specific source files mentioned.
 **When extending:** Follow existing patterns (context managers, async/await, type hints).
 **When optimizing:** Profile first (monitor.py metrics), then optimize hot paths (crawler_pool.py, api.py).
 **When troubleshooting memory:** Check browser pool cleanup, WebSocket connection tracking, Redis connection pooling.
 **When scaling issues:** Verify Docker mode (swarm vs compose), check nginx config if compose, review load balancer logs.
 ---
 **END OF ARCHITECTURE MAP**
 *Version: 1.0.0 | Last Updated: 2025-10-21 | Token-Optimized for AI Consumption*
--- a/deploy/docker/init.py
+++ b/deploy/docker/init.py
@@ -1 +0,0 @@
 # Deploy docker module
--- a/deploy/docker/api.py
+++ b/deploy/docker/api.py
@@ -67,7 +67,6 @@ async def handle_llm_qa(
    config: dict
 ) -> str:
    """Process QA using LLM with crawled content as context."""
    from crawler_pool import get_crawler
    try:
        if not url.startswith(('http://', 'https://')) and not url.startswith(("raw:", "raw://")):
            url = 'https://' + url
@@ -76,21 +75,15 @@ async def handle_llm_qa(
        if last_q_index != -1:
            url = url[:last_q_index]
-        # Get markdown content (use default config)
+        # Get markdown content
-        from utils import load_config
+        async with AsyncWebCrawler() as crawler:
-        cfg = load_config()
+            result = await crawler.arun(url)
-        browser_cfg = BrowserConfig(
+            if not result.success:
-            extra_args=cfg["crawler"]["browser"].get("extra_args", []),
+                raise HTTPException(
-            **cfg["crawler"]["browser"].get("kwargs", {}),
+                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-        )
+                    detail=result.error_message
-        crawler = await get_crawler(browser_cfg)
+                )
-        result = await crawler.arun(url)
+            content = result.markdown.fit_markdown or result.markdown.raw_markdown
        if not result.success:
            raise HTTPException(
                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                detail=result.error_message
            )
        content = result.markdown.fit_markdown or result.markdown.raw_markdown
        # Create prompt and get LLM response
        prompt = f"""Use the following content as context to answer the question.
@@ -279,32 +272,25 @@ async def handle_markdown_request(
        cache_mode = CacheMode.ENABLED if cache == "1" else CacheMode.WRITE_ONLY
-        from crawler_pool import get_crawler
+        async with AsyncWebCrawler() as crawler:
-        from utils import load_config as _load_config
+            result = await crawler.arun(
-        _cfg = _load_config()
+                url=decoded_url,
-        browser_cfg = BrowserConfig(
+                config=CrawlerRunConfig(
-            extra_args=_cfg["crawler"]["browser"].get("extra_args", []),
+                    markdown_generator=md_generator,
-            **_cfg["crawler"]["browser"].get("kwargs", {}),
+                    scraping_strategy=LXMLWebScrapingStrategy(),
-        )
+                    cache_mode=cache_mode
-        crawler = await get_crawler(browser_cfg)
+                )
        result = await crawler.arun(
            url=decoded_url,
            config=CrawlerRunConfig(
                markdown_generator=md_generator,
                scraping_strategy=LXMLWebScrapingStrategy(),
                cache_mode=cache_mode
            )
-        )
+            
            if not result.success:
                raise HTTPException(
                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                    detail=result.error_message
                )
-        if not result.success:
+            return (result.markdown.raw_markdown 
-            raise HTTPException(
+                   if filter_type == FilterType.RAW 
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                   else result.markdown.fit_markdown)
                detail=result.error_message
            )
        return (result.markdown.raw_markdown
               if filter_type == FilterType.RAW
               else result.markdown.fit_markdown)
    except Exception as e:
        logger.error(f"Markdown error: {str(e)}", exc_info=True)
@@ -518,22 +504,12 @@ async def handle_crawl_request(
    hooks_config: Optional[dict] = None
 ) -> dict:
    """Handle non-streaming crawl requests with optional hooks."""
    # Track request start
    request_id = f"req_{uuid4().hex[:8]}"
    try:
        from monitor import get_monitor
        await get_monitor().track_request_start(
            request_id, "/crawl", urls[0] if urls else "batch", browser_config
        )
    except:
        pass  # Monitor not critical
    start_mem_mb = _get_memory_mb() # <--- Get memory before
    start_time = time.time()
    mem_delta_mb = None
    peak_mem_mb = start_mem_mb
    hook_manager = None
-
+    
    try:
        urls = [('https://' + url) if not url.startswith(('http://', 'https://')) and not url.startswith(("raw:", "raw://")) else url for url in urls]
        browser_config = BrowserConfig.load(browser_config)
@@ -638,16 +614,7 @@ async def handle_crawl_request(
            "server_memory_delta_mb": mem_delta_mb,
            "server_peak_memory_mb": peak_mem_mb
        }
-
+        
        # Track request completion
        try:
            from monitor import get_monitor
            await get_monitor().track_request_end(
                request_id, success=True, pool_hit=True, status_code=200
            )
        except:
            pass
        # Add hooks information if hooks were used
        if hooks_config and hook_manager:
            from hook_manager import UserHookManager
@@ -676,16 +643,6 @@ async def handle_crawl_request(
    except Exception as e:
        logger.error(f"Crawl error: {str(e)}", exc_info=True)
        # Track request error
        try:
            from monitor import get_monitor
            await get_monitor().track_request_end(
                request_id, success=False, error=str(e), status_code=500
            )
        except:
            pass
        if 'crawler' in locals() and crawler.ready: # Check if crawler was initialized and started
            #  try:
            #      await crawler.close()
--- a/deploy/docker/docs/c4ai-code-context.md
+++ b/deploy/docker/docs/c4ai-code-context.md
--- a/deploy/docker/docs/c4ai-doc-context.md
+++ b/deploy/docker/docs/c4ai-doc-context.md
--- a/deploy/docker/cnode_cli.py
+++ b/deploy/docker/cnode_cli.py
@@ -1,492 +0,0 @@
 """
 Crawl4AI Server CLI Commands
 Provides `cnode` command group for Docker orchestration.
 """
 import click
 import anyio
 from rich.console import Console
 from rich.table import Table
 from rich.panel import Panel
 from rich.prompt import Confirm
 from deploy.docker.server_manager import ServerManager
 console = Console()
@click.group()
 def cli():
    """Manage Crawl4AI Docker server instances
    \b
    One-command deployment with automatic scaling:
      • Single container for development (N=1)
      • Docker Swarm for production with built-in load balancing (N>1)
      • Docker Compose + Nginx as fallback (N>1)
    \b
    Examples:
      cnode start                    # Single container on port 11235
      cnode start --replicas 3       # Auto-detect Swarm or Compose
      cnode start -r 5 --port 8080   # 5 replicas on custom port
      cnode status                   # Check current deployment
      cnode scale 10                 # Scale to 10 replicas
      cnode stop                     # Stop and cleanup
    """
    pass
@cli.command("start")
@click.option(
    "--replicas", "-r",
    type=int,
    default=1,
    help="Number of container replicas (default: 1)"
 )
@click.option(
    "--mode",
    type=click.Choice(["auto", "single", "swarm", "compose"]),
    default="auto",
    help="Deployment mode (default: auto-detect)"
 )
@click.option(
    "--port", "-p",
    type=int,
    default=11235,
    help="External port to expose (default: 11235)"
 )
@click.option(
    "--env-file",
    type=click.Path(exists=True),
    help="Path to environment file"
 )
@click.option(
    "--image",
    default="unclecode/crawl4ai:latest",
    help="Docker image to use (default: unclecode/crawl4ai:latest)"
 )
 def start_cmd(replicas: int, mode: str, port: int, env_file: str, image: str):
    """Start Crawl4AI server with automatic orchestration.
    Deployment modes:
    - auto: Automatically choose best mode (default)
    - single: Single container (N=1 only)
    - swarm: Docker Swarm with built-in load balancing
    - compose: Docker Compose + Nginx reverse proxy
    The server will:
    1. Check if Docker is running
    2. Validate port availability
    3. Pull image if needed
    4. Start container(s) with health checks
    5. Save state for management
    Examples:
        # Development: single container
        cnode start
        # Production: 5 replicas with Swarm
        cnode start --replicas 5
        # Custom configuration
        cnode start -r 3 --port 8080 --env-file .env.prod
    """
    manager = ServerManager()
    console.print(Panel(
        f"[cyan]Starting Crawl4AI Server[/cyan]\n\n"
        f"Replicas: [yellow]{replicas}[/yellow]\n"
        f"Mode: [yellow]{mode}[/yellow]\n"
        f"Port: [yellow]{port}[/yellow]\n"
        f"Image: [yellow]{image}[/yellow]",
        title="Server Start",
        border_style="cyan"
    ))
    with console.status("[cyan]Starting server..."):
        async def _start():
            return await manager.start(
                replicas=replicas,
                mode=mode,
                port=port,
                env_file=env_file,
                image=image
            )
        result = anyio.run(_start)
    if result["success"]:
        console.print(Panel(
            f"[green]✓ Server started successfully![/green]\n\n"
            f"Mode: [cyan]{result.get('state_data', {}).get('mode', mode)}[/cyan]\n"
            f"URL: [bold]http://localhost:{port}[/bold]\n"
            f"Health: [bold]http://localhost:{port}/health[/bold]\n"
            f"Monitor: [bold]http://localhost:{port}/monitor[/bold]",
            title="Server Running",
            border_style="green"
        ))
    else:
        error_msg = result.get("error", result.get("message", "Unknown error"))
        console.print(Panel(
            f"[red]✗ Failed to start server[/red]\n\n"
            f"{error_msg}",
            title="Error",
            border_style="red"
        ))
        if "already running" in error_msg.lower():
            console.print("\n[yellow]Hint: Use 'cnode status' to check current deployment[/yellow]")
            console.print("[yellow]      Use 'cnode stop' to stop existing server[/yellow]")
@cli.command("status")
 def status_cmd():
    """Show current server status and deployment info.
    Displays:
    - Running state (up/down)
    - Deployment mode (single/swarm/compose)
    - Number of replicas
    - Port mapping
    - Uptime
    - Image version
    Example:
        cnode status
    """
    manager = ServerManager()
    async def _status():
        return await manager.status()
    result = anyio.run(_status)
    if result["running"]:
        table = Table(title="Crawl4AI Server Status", border_style="green")
        table.add_column("Property", style="cyan")
        table.add_column("Value", style="green")
        table.add_row("Status", "🟢 Running")
        table.add_row("Mode", result["mode"])
        table.add_row("Replicas", str(result.get("replicas", 1)))
        table.add_row("Port", str(result.get("port", 11235)))
        table.add_row("Image", result.get("image", "unknown"))
        table.add_row("Uptime", result.get("uptime", "unknown"))
        table.add_row("Started", result.get("started_at", "unknown"))
        console.print(table)
        console.print(f"\n[green]✓ Server is healthy[/green]")
        console.print(f"[dim]Access: http://localhost:{result.get('port', 11235)}[/dim]")
    else:
        console.print(Panel(
            f"[yellow]No server is currently running[/yellow]\n\n"
            f"Use 'cnode start' to launch a server",
            title="Server Status",
            border_style="yellow"
        ))
@cli.command("stop")
@click.option(
    "--remove-volumes",
    is_flag=True,
    help="Remove associated volumes (WARNING: deletes data)"
 )
 def stop_cmd(remove_volumes: bool):
    """Stop running Crawl4AI server and cleanup resources.
    This will:
    1. Stop all running containers/services
    2. Remove containers
    3. Optionally remove volumes (--remove-volumes)
    4. Clean up state files
    WARNING: Use --remove-volumes with caution as it will delete
    persistent data including Redis databases and logs.
    Examples:
        # Stop server, keep volumes
        cnode stop
        # Stop and remove all data
        cnode stop --remove-volumes
    """
    manager = ServerManager()
    # Confirm if removing volumes
    if remove_volumes:
        if not Confirm.ask(
            "[red]⚠️  This will delete all server data including Redis databases. Continue?[/red]"
        ):
            console.print("[yellow]Cancelled[/yellow]")
            return
    with console.status("[cyan]Stopping server..."):
        async def _stop():
            return await manager.stop(remove_volumes=remove_volumes)
        result = anyio.run(_stop)
    if result["success"]:
        console.print(Panel(
            f"[green]✓ Server stopped successfully[/green]\n\n"
            f"{result.get('message', 'All resources cleaned up')}",
            title="Server Stopped",
            border_style="green"
        ))
    else:
        console.print(Panel(
            f"[red]✗ Error stopping server[/red]\n\n"
            f"{result.get('error', result.get('message', 'Unknown error'))}",
            title="Error",
            border_style="red"
        ))
@cli.command("scale")
@click.argument("replicas", type=int)
 def scale_cmd(replicas: int):
    """Scale server to specified number of replicas.
    Only works with Swarm or Compose modes. Single container
    mode cannot be scaled (must stop and restart with --replicas).
    Scaling is live and does not require downtime. The load
    balancer will automatically distribute traffic to new replicas.
    Examples:
        # Scale up to 10 replicas
        cnode scale 10
        # Scale down to 2 replicas
        cnode scale 2
        # Scale to 1 (minimum)
        cnode scale 1
    """
    if replicas < 1:
        console.print("[red]Error: Replicas must be at least 1[/red]")
        return
    manager = ServerManager()
    with console.status(f"[cyan]Scaling to {replicas} replicas..."):
        async def _scale():
            return await manager.scale(replicas=replicas)
        result = anyio.run(_scale)
    if result["success"]:
        console.print(Panel(
            f"[green]✓ Scaled successfully[/green]\n\n"
            f"New replica count: [bold]{replicas}[/bold]\n"
            f"Mode: [cyan]{result.get('mode')}[/cyan]",
            title="Scaling Complete",
            border_style="green"
        ))
    else:
        error_msg = result.get("error", result.get("message", "Unknown error"))
        console.print(Panel(
            f"[red]✗ Scaling failed[/red]\n\n"
            f"{error_msg}",
            title="Error",
            border_style="red"
        ))
        if "single container" in error_msg.lower():
            console.print("\n[yellow]Hint: For single container mode:[/yellow]")
            console.print("[yellow]  1. cnode stop[/yellow]")
            console.print(f"[yellow]  2. cnode start --replicas {replicas}[/yellow]")
@cli.command("logs")
@click.option(
    "--follow", "-f",
    is_flag=True,
    help="Follow log output (like tail -f)"
 )
@click.option(
    "--tail",
    type=int,
    default=100,
    help="Number of lines to show (default: 100)"
 )
 def logs_cmd(follow: bool, tail: int):
    """View server logs.
    Shows logs from running containers/services. Use --follow
    to stream logs in real-time.
    Examples:
        # Show last 100 lines
        cnode logs
        # Show last 500 lines
        cnode logs --tail 500
        # Follow logs in real-time
        cnode logs --follow
        # Combine options
        cnode logs -f --tail 50
    """
    manager = ServerManager()
    async def _logs():
        return await manager.logs(follow=follow, tail=tail)
    output = anyio.run(_logs)
    console.print(output)
@cli.command("cleanup")
@click.option(
    "--force",
    is_flag=True,
    help="Force cleanup even if state file doesn't exist"
 )
 def cleanup_cmd(force: bool):
    """Force cleanup of all Crawl4AI Docker resources.
    Stops and removes all containers, networks, and optionally volumes.
    Useful when server is stuck or state is corrupted.
    Examples:
        # Clean up everything
        cnode cleanup
        # Force cleanup (ignore state file)
        cnode cleanup --force
    """
    manager = ServerManager()
    console.print(Panel(
        f"[yellow]⚠️  Cleaning up Crawl4AI Docker resources[/yellow]\n\n"
        f"This will stop and remove:\n"
        f"- All Crawl4AI containers\n"
        f"- Nginx load balancer\n"
        f"- Redis instance\n"
        f"- Docker networks\n"
        f"- State files",
        title="Cleanup",
        border_style="yellow"
    ))
    if not force and not Confirm.ask("[yellow]Continue with cleanup?[/yellow]"):
        console.print("[yellow]Cancelled[/yellow]")
        return
    with console.status("[cyan]Cleaning up resources..."):
        async def _cleanup():
            return await manager.cleanup(force=force)
        result = anyio.run(_cleanup)
    if result["success"]:
        console.print(Panel(
            f"[green]✓ Cleanup completed successfully[/green]\n\n"
            f"Removed: {result.get('removed', 0)} containers\n"
            f"{result.get('message', 'All resources cleaned up')}",
            title="Cleanup Complete",
            border_style="green"
        ))
    else:
        console.print(Panel(
            f"[yellow]⚠️  Partial cleanup[/yellow]\n\n"
            f"{result.get('message', 'Some resources may still exist')}",
            title="Cleanup Status",
            border_style="yellow"
        ))
@cli.command("restart")
@click.option(
    "--replicas", "-r",
    type=int,
    help="New replica count (optional)"
 )
 def restart_cmd(replicas: int):
    """Restart server (stop then start with same config).
    Preserves existing configuration unless overridden with options.
    Useful for applying image updates or recovering from errors.
    Examples:
        # Restart with same configuration
        cnode restart
        # Restart and change replica count
        cnode restart --replicas 5
    """
    manager = ServerManager()
    # Get current state
    async def _get_status():
        return await manager.status()
    current = anyio.run(_get_status)
    if not current["running"]:
        console.print("[yellow]No server is running. Use 'cnode start' instead.[/yellow]")
        return
    # Extract current config
    current_replicas = current.get("replicas", 1)
    current_port = current.get("port", 11235)
    current_image = current.get("image", "unclecode/crawl4ai:latest")
    current_mode = current.get("mode", "auto")
    # Override with CLI args
    new_replicas = replicas if replicas is not None else current_replicas
    console.print(Panel(
        f"[cyan]Restarting Crawl4AI Server[/cyan]\n\n"
        f"Replicas: [yellow]{current_replicas}[/yellow] → [green]{new_replicas}[/green]\n"
        f"Port: [yellow]{current_port}[/yellow]\n"
        f"Mode: [yellow]{current_mode}[/yellow]",
        title="Server Restart",
        border_style="cyan"
    ))
    # Stop current
    with console.status("[cyan]Stopping current server..."):
        async def _stop_server():
            return await manager.stop(remove_volumes=False)
        stop_result = anyio.run(_stop_server)
    if not stop_result["success"]:
        console.print(f"[red]Failed to stop server: {stop_result.get('error')}[/red]")
        return
    # Start new
    with console.status("[cyan]Starting server..."):
        async def _start_server():
            return await manager.start(
                replicas=new_replicas,
                mode="auto",
                port=current_port,
                image=current_image
            )
        start_result = anyio.run(_start_server)
    if start_result["success"]:
        console.print(Panel(
            f"[green]✓ Server restarted successfully![/green]\n\n"
            f"URL: [bold]http://localhost:{current_port}[/bold]",
            title="Restart Complete",
            border_style="green"
        ))
    else:
        console.print(Panel(
            f"[red]✗ Failed to restart server[/red]\n\n"
            f"{start_result.get('error', 'Unknown error')}",
            title="Error",
            border_style="red"
        ))
 def main():
    """Entry point for cnode CLI"""
    cli()
 if __name__ == "__main__":
    main()
 # Test comment
--- a/deploy/docker/config.yml
+++ b/deploy/docker/config.yml
@@ -3,7 +3,7 @@ app:
  title: "Crawl4AI API"
  version: "1.0.0"
  host: "0.0.0.0"
-  port: 11235
+  port: 11234
  reload: False
  workers: 1
  timeout_keep_alive: 300
@@ -61,7 +61,7 @@ crawler:
    batch_process: 300.0  # Timeout for batch processing
  pool:
    max_pages: 40                          # ← GLOBAL_SEM permits
-    idle_ttl_sec: 300                     # ← 30 min janitor cutoff
+    idle_ttl_sec: 1800                     # ← 30 min janitor cutoff
  browser:
    kwargs:
      headless: true
--- a/deploy/docker/crawler_pool.py
+++ b/deploy/docker/crawler_pool.py
@@ -1,170 +1,72 @@
-# crawler_pool.py - Smart browser pool with tiered management
+# crawler_pool.py  (new file)
-import asyncio, json, hashlib, time
+import asyncio, json, hashlib, time, psutil
 from contextlib import suppress
-from typing import Dict, Optional
+from typing import Dict
-from crawl4ai import AsyncWebCrawler, BrowserConfig
+from crawl4ai import AsyncWebCrawler, BrowserConfig, BrowserAdapter
-from utils import load_config, get_container_memory_percent
+from typing import Dict ,Optional
-import logging
+from utils import load_config 
 logger = logging.getLogger(__name__)
 CONFIG = load_config()
-# Pool tiers
+POOL: Dict[str, AsyncWebCrawler] = {}
 PERMANENT: Optional[AsyncWebCrawler] = None  # Always-ready default browser
 HOT_POOL: Dict[str, AsyncWebCrawler] = {}    # Frequent configs
 COLD_POOL: Dict[str, AsyncWebCrawler] = {}   # Rare configs
 LAST_USED: Dict[str, float] = {}
 USAGE_COUNT: Dict[str, int] = {}
 LOCK = asyncio.Lock()
-# Config
+MEM_LIMIT  = CONFIG.get("crawler", {}).get("memory_threshold_percent", 95.0)   # % RAM – refuse new browsers above this
-MEM_LIMIT = CONFIG.get("crawler", {}).get("memory_threshold_percent", 95.0)
+IDLE_TTL  = CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 1800)   # close if unused for 30 min
 BASE_IDLE_TTL = CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 300)
 DEFAULT_CONFIG_SIG = None  # Cached sig for default config
-def _sig(cfg: BrowserConfig) -> str:
+
-    """Generate config signature."""
+def _sig(cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None) -> str:
-    payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",",":"))
+    try:
        config_payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",", ":"))
    except (TypeError, ValueError):
        # Fallback to string representation if JSON serialization fails
        config_payload = str(cfg.to_dict())
    adapter_name = adapter.__class__.__name__ if adapter else "PlaywrightAdapter"
    payload = f"{config_payload}:{adapter_name}"
    return hashlib.sha1(payload.encode()).hexdigest()
 def _is_default_config(sig: str) -> bool:
    """Check if config matches default."""
    return sig == DEFAULT_CONFIG_SIG
 async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler:
    """Get crawler from pool with tiered strategy."""
    sig = _sig(cfg)
    async with LOCK:
        # Check permanent browser for default config
        if PERMANENT and _is_default_config(sig):
            LAST_USED[sig] = time.time()
            USAGE_COUNT[sig] = USAGE_COUNT.get(sig, 0) + 1
            logger.info("🔥 Using permanent browser")
            return PERMANENT
        # Check hot pool
        if sig in HOT_POOL:
            LAST_USED[sig] = time.time()
            USAGE_COUNT[sig] = USAGE_COUNT.get(sig, 0) + 1
            logger.info(f"♨️  Using hot pool browser (sig={sig[:8]})")
            return HOT_POOL[sig]
        # Check cold pool (promote to hot if used 3+ times)
        if sig in COLD_POOL:
            LAST_USED[sig] = time.time()
            USAGE_COUNT[sig] = USAGE_COUNT.get(sig, 0) + 1
            if USAGE_COUNT[sig] >= 3:
                logger.info(f"⬆️  Promoting to hot pool (sig={sig[:8]}, count={USAGE_COUNT[sig]})")
                HOT_POOL[sig] = COLD_POOL.pop(sig)
                # Track promotion in monitor
                try:
                    from monitor import get_monitor
                    await get_monitor().track_janitor_event("promote", sig, {"count": USAGE_COUNT[sig]})
                except:
                    pass
                return HOT_POOL[sig]
            logger.info(f"❄️  Using cold pool browser (sig={sig[:8]})")
            return COLD_POOL[sig]
        # Memory check before creating new
        mem_pct = get_container_memory_percent()
        if mem_pct >= MEM_LIMIT:
            logger.error(f"💥 Memory pressure: {mem_pct:.1f}% >= {MEM_LIMIT}%")
            raise MemoryError(f"Memory at {mem_pct:.1f}%, refusing new browser")
        # Create new in cold pool
        logger.info(f"🆕 Creating new browser in cold pool (sig={sig[:8]}, mem={mem_pct:.1f}%)")
        crawler = AsyncWebCrawler(config=cfg, thread_safe=False)
        await crawler.start()
        COLD_POOL[sig] = crawler
        LAST_USED[sig] = time.time()
        USAGE_COUNT[sig] = 1
        return crawler
 async def init_permanent(cfg: BrowserConfig):
    """Initialize permanent default browser."""
    global PERMANENT, DEFAULT_CONFIG_SIG
    async with LOCK:
        if PERMANENT:
            return
        DEFAULT_CONFIG_SIG = _sig(cfg)
        logger.info("🔥 Creating permanent default browser")
        PERMANENT = AsyncWebCrawler(config=cfg, thread_safe=False)
        await PERMANENT.start()
        LAST_USED[DEFAULT_CONFIG_SIG] = time.time()
        USAGE_COUNT[DEFAULT_CONFIG_SIG] = 0
 async def get_crawler(
    cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None
 ) -> AsyncWebCrawler:
    sig = None
    try:
        sig = _sig(cfg)
        async with LOCK:
            if sig in POOL:
                LAST_USED[sig] = time.time();  
                return POOL[sig]
            if psutil.virtual_memory().percent >= MEM_LIMIT:
                raise MemoryError("RAM pressure – new browser denied")
            crawler = AsyncWebCrawler(config=cfg, thread_safe=False)
            await crawler.start()
            POOL[sig] = crawler; LAST_USED[sig] = time.time()
            return crawler
    except MemoryError as e:
        raise MemoryError(f"RAM pressure – new browser denied: {e}")
    except Exception as e:
        raise RuntimeError(f"Failed to start browser: {e}")
    finally:
        if sig:
            if sig in POOL:
                LAST_USED[sig] = time.time()
            else:
                # If we failed to start the browser, we should remove it from the pool
                POOL.pop(sig, None)
                LAST_USED.pop(sig, None)
        # If we failed to start the browser, we should remove it from the pool
 async def close_all():
    """Close all browsers."""
    async with LOCK:
-        tasks = []
+        await asyncio.gather(*(c.close() for c in POOL.values()), return_exceptions=True)
-        if PERMANENT:
+        POOL.clear(); LAST_USED.clear()
            tasks.append(PERMANENT.close())
        tasks.extend([c.close() for c in HOT_POOL.values()])
        tasks.extend([c.close() for c in COLD_POOL.values()])
        await asyncio.gather(*tasks, return_exceptions=True)
        HOT_POOL.clear()
        COLD_POOL.clear()
        LAST_USED.clear()
        USAGE_COUNT.clear()
 async def janitor():
    """Adaptive cleanup based on memory pressure."""
    while True:
-        mem_pct = get_container_memory_percent()
+        await asyncio.sleep(60)
        # Adaptive intervals and TTLs
        if mem_pct > 80:
            interval, cold_ttl, hot_ttl = 10, 30, 120
        elif mem_pct > 60:
            interval, cold_ttl, hot_ttl = 30, 60, 300
        else:
            interval, cold_ttl, hot_ttl = 60, BASE_IDLE_TTL, BASE_IDLE_TTL * 2
        await asyncio.sleep(interval)
        now = time.time()
        async with LOCK:
-            # Clean cold pool
+            for sig, crawler in list(POOL.items()):
-            for sig in list(COLD_POOL.keys()):
+                if now - LAST_USED[sig] > IDLE_TTL:
-                if now - LAST_USED.get(sig, now) > cold_ttl:
+                    with suppress(Exception): await crawler.close()
-                    idle_time = now - LAST_USED[sig]
+                    POOL.pop(sig, None); LAST_USED.pop(sig, None)
                    logger.info(f"🧹 Closing cold browser (sig={sig[:8]}, idle={idle_time:.0f}s)")
                    with suppress(Exception):
                        await COLD_POOL[sig].close()
                    COLD_POOL.pop(sig, None)
                    LAST_USED.pop(sig, None)
                    USAGE_COUNT.pop(sig, None)
                    # Track in monitor
                    try:
                        from monitor import get_monitor
                        await get_monitor().track_janitor_event("close_cold", sig, {"idle_seconds": int(idle_time), "ttl": cold_ttl})
                    except:
                        pass
            # Clean hot pool (more conservative)
            for sig in list(HOT_POOL.keys()):
                if now - LAST_USED.get(sig, now) > hot_ttl:
                    idle_time = now - LAST_USED[sig]
                    logger.info(f"🧹 Closing hot browser (sig={sig[:8]}, idle={idle_time:.0f}s)")
                    with suppress(Exception):
                        await HOT_POOL[sig].close()
                    HOT_POOL.pop(sig, None)
                    LAST_USED.pop(sig, None)
                    USAGE_COUNT.pop(sig, None)
                    # Track in monitor
                    try:
                        from monitor import get_monitor
                        await get_monitor().track_janitor_event("close_hot", sig, {"idle_seconds": int(idle_time), "ttl": hot_ttl})
                    except:
                        pass
            # Log pool stats
            if mem_pct > 60:
                logger.info(f"📊 Pool: hot={len(HOT_POOL)}, cold={len(COLD_POOL)}, mem={mem_pct:.1f}%")
--- a/deploy/docker/docs/ARCHITECTURE.md
+++ b/deploy/docker/docs/ARCHITECTURE.md
--- a/deploy/docker/docs/DOCKER_ORCHESTRATION.md
+++ b/deploy/docker/docs/DOCKER_ORCHESTRATION.md
--- a/deploy/docker/docs/MULTI_CONTAINER_ARCHITECTURE.md
+++ b/deploy/docker/docs/MULTI_CONTAINER_ARCHITECTURE.md
--- a/deploy/docker/docs/STRESS_TEST_PIPELINE.md
+++ b/deploy/docker/docs/STRESS_TEST_PIPELINE.md
@@ -1,241 +0,0 @@
 # Crawl4AI Docker Memory & Pool Optimization - Implementation Log
 ## Critical Issues Identified
 ### Memory Management
 - **Host vs Container**: `psutil.virtual_memory()` reported host memory, not container limits
 - **Browser Pooling**: No pool reuse - every endpoint created new browsers
 - **Warmup Waste**: Permanent browser sat idle with mismatched config signature
 - **Idle Cleanup**: 30min TTL too long, janitor ran every 60s
 - **Endpoint Inconsistency**: 75% of endpoints bypassed pool (`/md`, `/html`, `/screenshot`, `/pdf`, `/execute_js`, `/llm`)
 ### Pool Design Flaws
 - **Config Mismatch**: Permanent browser used `config.yml` args, endpoints used empty `BrowserConfig()`
 - **Logging Level**: Pool hit markers at DEBUG, invisible with INFO logging
 ## Implementation Changes
 ### 1. Container-Aware Memory Detection (`utils.py`)
 ```python
 def get_container_memory_percent() -> float:
    # Try cgroup v2 → v1 → fallback to psutil
    # Reads /sys/fs/cgroup/memory.{current,max} OR memory/memory.{usage,limit}_in_bytes
 ```
 ### 2. Smart Browser Pool (`crawler_pool.py`)
 **3-Tier System:**
 - **PERMANENT**: Always-ready default browser (never cleaned)
 - **HOT_POOL**: Configs used 3+ times (longer TTL)
 - **COLD_POOL**: New/rare configs (short TTL)
 **Key Functions:**
 - `get_crawler(cfg)`: Check permanent → hot → cold → create new
 - `init_permanent(cfg)`: Initialize permanent at startup
 - `janitor()`: Adaptive cleanup (10s/30s/60s intervals based on memory)
 - `_sig(cfg)`: SHA1 hash of config dict for pool keys
 **Logging Fix**: Changed `logger.debug()` → `logger.info()` for pool hits
 ### 3. Endpoint Unification
 **Helper Function** (`server.py`):
 ```python
 def get_default_browser_config() -> BrowserConfig:
    return BrowserConfig(
        extra_args=config["crawler"]["browser"].get("extra_args", []),
        **config["crawler"]["browser"].get("kwargs", {}),
    )
 ```
 **Migrated Endpoints:**
 - `/html`, `/screenshot`, `/pdf`, `/execute_js` → use `get_default_browser_config()`
 - `handle_llm_qa()`, `handle_markdown_request()` → same
 **Result**: All endpoints now hit permanent browser pool
 ### 4. Config Updates (`config.yml`)
 - `idle_ttl_sec: 1800` → `300` (30min → 5min base TTL)
 - `port: 11234` → `11235` (fixed mismatch with Gunicorn)
 ### 5. Lifespan Fix (`server.py`)
 ```python
 await init_permanent(BrowserConfig(
    extra_args=config["crawler"]["browser"].get("extra_args", []),
    **config["crawler"]["browser"].get("kwargs", {}),
 ))
 ```
 Permanent browser now matches endpoint config signatures
 ## Test Results
 ### Test 1: Basic Health
 - 10 requests to `/health`
 - **Result**: 100% success, avg 3ms latency
 - **Baseline**: Container starts in ~5s, 270 MB idle
 ### Test 2: Memory Monitoring
 - 20 requests with Docker stats tracking
 - **Result**: 100% success, no memory leak (-0.2 MB delta)
 - **Baseline**: 269.7 MB container overhead
 ### Test 3: Pool Validation
 - 30 requests to `/html` endpoint
 - **Result**: **100% permanent browser hits**, 0 new browsers created
 - **Memory**: 287 MB baseline → 396 MB active (+109 MB)
 - **Latency**: Avg 4s (includes network to httpbin.org)
 ### Test 4: Concurrent Load
 - Light (10) → Medium (50) → Heavy (100) concurrent
 - **Total**: 320 requests
 - **Result**: 100% success, **320/320 permanent hits**, 0 new browsers
 - **Memory**: 269 MB → peak 1533 MB → final 993 MB
 - **Latency**: P99 at 100 concurrent = 34s (expected with single browser)
 ### Test 5: Pool Stress (Mixed Configs)
 - 20 requests with 4 different viewport configs
 - **Result**: 4 new browsers, 4 cold hits, **4 promotions to hot**, 8 hot hits
 - **Reuse Rate**: 60% (12 pool hits / 20 requests)
 - **Memory**: 270 MB → 928 MB peak (+658 MB = ~165 MB per browser)
 - **Proves**: Cold → hot promotion at 3 uses working perfectly
 ### Test 6: Multi-Endpoint
 - 10 requests each: `/html`, `/screenshot`, `/pdf`, `/crawl`
 - **Result**: 100% success across all 4 endpoints
 - **Latency**: 5-8s avg (PDF slowest at 7.2s)
 ### Test 7: Cleanup Verification
 - 20 requests (load spike) → 90s idle
 - **Memory**: 269 MB → peak 1107 MB → final 780 MB
 - **Recovery**: 327 MB (39%) - partial cleanup
 - **Note**: Hot pool browsers persist (by design), janitor working correctly
 ## Performance Metrics
 | Metric | Before | After | Improvement |
 |--------|--------|-------|-------------|
 | Pool Reuse | 0% | 100% (default config) | ∞ |
 | Memory Leak | Unknown | 0 MB/cycle | Stable |
 | Browser Reuse | No | Yes | ~3-5s saved per request |
 | Idle Memory | 500-700 MB × N | 270-400 MB | 10x reduction |
 | Concurrent Capacity | ~20 | 100+ | 5x |
 ## Key Learnings
 1. **Config Signature Matching**: Permanent browser MUST match endpoint default config exactly (SHA1 hash)
 2. **Logging Levels**: Pool diagnostics need INFO level, not DEBUG
 3. **Memory in Docker**: Must read cgroup files, not host metrics
 4. **Janitor Timing**: 60s interval adequate, but TTLs should be short (5min) for cold pool
 5. **Hot Promotion**: 3-use threshold works well for production patterns
 6. **Memory Per Browser**: ~150-200 MB per Chromium instance with headless + text_mode
 ## Test Infrastructure
 **Location**: `deploy/docker/tests/`
 **Dependencies**: `httpx`, `docker` (Python SDK)
 **Pattern**: Sequential build - each test adds one capability
 **Files**:
 - `test_1_basic.py`: Health check + container lifecycle
 - `test_2_memory.py`: + Docker stats monitoring
 - `test_3_pool.py`: + Log analysis for pool markers
 - `test_4_concurrent.py`: + asyncio.Semaphore for concurrency control
 - `test_5_pool_stress.py`: + Config variants (viewports)
 - `test_6_multi_endpoint.py`: + Multiple endpoint testing
 - `test_7_cleanup.py`: + Time-series memory tracking for janitor
 **Run Pattern**:
 ```bash
 cd deploy/docker/tests
 pip install -r requirements.txt
 # Rebuild after code changes:
 cd /path/to/repo && docker buildx build -t crawl4ai-local:latest --load .
 # Run test:
 python test_N_name.py
 ```
 ## Architecture Decisions
 **Why Permanent Browser?**
 - 90% of requests use default config → single browser serves most traffic
 - Eliminates 3-5s startup overhead per request
 **Why 3-Tier Pool?**
 - Permanent: Zero cost for common case
 - Hot: Amortized cost for frequent variants
 - Cold: Lazy allocation for rare configs
 **Why Adaptive Janitor?**
 - Memory pressure triggers aggressive cleanup
 - Low memory allows longer TTLs for better reuse
 **Why Not Close After Each Request?**
 - Browser startup: 3-5s overhead
 - Pool reuse: <100ms overhead
 - Net: 30-50x faster
 ## Future Optimizations
 1. **Request Queuing**: When at capacity, queue instead of reject
 2. **Pre-warming**: Predict common configs, pre-create browsers
 3. **Metrics Export**: Prometheus metrics for pool efficiency
 4. **Config Normalization**: Group similar viewports (e.g., 1920±50 → 1920)
 ## Critical Code Paths
 **Browser Acquisition** (`crawler_pool.py:34-78`):
 ```
 get_crawler(cfg) →
  _sig(cfg) →
  if sig == DEFAULT_CONFIG_SIG → PERMANENT
  elif sig in HOT_POOL → HOT_POOL[sig]
  elif sig in COLD_POOL → promote if count >= 3
  else → create new in COLD_POOL
 ```
 **Janitor Loop** (`crawler_pool.py:107-146`):
 ```
 while True:
  mem% = get_container_memory_percent()
  if mem% > 80: interval=10s, cold_ttl=30s
  elif mem% > 60: interval=30s, cold_ttl=60s
  else: interval=60s, cold_ttl=300s
  sleep(interval)
  close idle browsers (COLD then HOT)
 ```
 **Endpoint Pattern** (`server.py` example):
 ```python
@app.post("/html")
 async def generate_html(...):
    from crawler_pool import get_crawler
    crawler = await get_crawler(get_default_browser_config())
    results = await crawler.arun(url=body.url, config=cfg)
    # No crawler.close() - returned to pool
 ```
 ## Debugging Tips
 **Check Pool Activity**:
 ```bash
 docker logs crawl4ai-test | grep -E "(🔥|♨️|❄️|🆕|⬆️)"
 ```
 **Verify Config Signature**:
 ```python
 from crawl4ai import BrowserConfig
 import json, hashlib
 cfg = BrowserConfig(...)
 sig = hashlib.sha1(json.dumps(cfg.to_dict(), sort_keys=True).encode()).hexdigest()
 print(sig[:8])  # Compare with logs
 ```
 **Monitor Memory**:
 ```bash
 docker stats crawl4ai-test
 ```
 ## Known Limitations
 - **Mac Docker Stats**: CPU metrics unreliable, memory works
 - **PDF Generation**: Slowest endpoint (~7s), no optimization yet
 - **Hot Pool Persistence**: May hold memory longer than needed (trade-off for performance)
 - **Janitor Lag**: Up to 60s before cleanup triggers in low-memory scenarios
--- a/deploy/docker/monitor.py
+++ b/deploy/docker/monitor.py
@@ -1,663 +0,0 @@
 # monitor.py - Real-time monitoring stats with Redis persistence
 import time
 import json
 import asyncio
 from typing import Dict, List, Optional
 from datetime import datetime, timezone
 from collections import deque
 from dataclasses import dataclass
 from redis import asyncio as aioredis
 from utils import get_container_memory_percent
 import psutil
 import logging
 logger = logging.getLogger(__name__)
 # ========== Configuration ==========
@dataclass
 class RedisTTLConfig:
    """Redis TTL configuration (in seconds).
    Configures how long different types of monitoring data are retained in Redis.
    Adjust based on your monitoring needs and Redis memory constraints.
    """
    active_requests: int = 300  # 5 minutes - short-lived active request data
    completed_requests: int = 3600  # 1 hour - recent completed requests
    janitor_events: int = 3600  # 1 hour - browser cleanup events
    errors: int = 3600  # 1 hour - error logs
    endpoint_stats: int = 86400  # 24 hours - aggregated endpoint statistics
    heartbeat: int = 60  # 1 minute - container heartbeat (2x the 30s interval)
    @classmethod
    def from_env(cls) -> 'RedisTTLConfig':
        """Load TTL configuration from environment variables."""
        import os
        return cls(
            active_requests=int(os.getenv('REDIS_TTL_ACTIVE_REQUESTS', 300)),
            completed_requests=int(os.getenv('REDIS_TTL_COMPLETED_REQUESTS', 3600)),
            janitor_events=int(os.getenv('REDIS_TTL_JANITOR_EVENTS', 3600)),
            errors=int(os.getenv('REDIS_TTL_ERRORS', 3600)),
            endpoint_stats=int(os.getenv('REDIS_TTL_ENDPOINT_STATS', 86400)),
            heartbeat=int(os.getenv('REDIS_TTL_HEARTBEAT', 60)),
        )
 class MonitorStats:
    """Tracks real-time server stats with Redis persistence."""
    def __init__(self, redis: aioredis.Redis, ttl_config: Optional[RedisTTLConfig] = None):
        self.redis = redis
        self.ttl = ttl_config or RedisTTLConfig.from_env()
        self.start_time = time.time()
        # Get container ID for Redis keys
        from utils import get_container_id
        self.container_id = get_container_id()
        # In-memory queues (fast reads, Redis backup)
        self.active_requests: Dict[str, Dict] = {}  # id -> request info
        self.completed_requests: deque = deque(maxlen=100)  # Last 100
        self.janitor_events: deque = deque(maxlen=100)
        self.errors: deque = deque(maxlen=100)
        # Endpoint stats (persisted in Redis)
        self.endpoint_stats: Dict[str, Dict] = {}  # endpoint -> {count, total_time, errors, ...}
        # Background persistence queue (max 10 pending persist requests)
        self._persist_queue: asyncio.Queue = asyncio.Queue(maxsize=10)
        self._persist_worker_task: Optional[asyncio.Task] = None
        # Heartbeat task for container discovery
        self._heartbeat_task: Optional[asyncio.Task] = None
        # Timeline data (5min window, 5s resolution = 60 points)
        self.memory_timeline: deque = deque(maxlen=60)
        self.requests_timeline: deque = deque(maxlen=60)
        self.browser_timeline: deque = deque(maxlen=60)
    async def track_request_start(self, request_id: str, endpoint: str, url: str, config: Dict = None):
        """Track new request start."""
        req_info = {
            "id": request_id,
            "endpoint": endpoint,
            "url": url[:100],  # Truncate long URLs
            "start_time": time.time(),
            "config_sig": config.get("sig", "default") if config else "default",
            "mem_start": psutil.Process().memory_info().rss / (1024 * 1024),
            "container_id": self.container_id
        }
        self.active_requests[request_id] = req_info
        # Persist to Redis
        await self._persist_active_requests()
        # Increment endpoint counter
        if endpoint not in self.endpoint_stats:
            self.endpoint_stats[endpoint] = {
                "count": 0, "total_time": 0, "errors": 0,
                "pool_hits": 0, "success": 0
            }
        self.endpoint_stats[endpoint]["count"] += 1
        # Queue persistence (handled by background worker)
        try:
            self._persist_queue.put_nowait(True)
        except asyncio.QueueFull:
            logger.warning("Persistence queue full, skipping")
    async def track_request_end(self, request_id: str, success: bool, error: str = None,
                               pool_hit: bool = True, status_code: int = 200):
        """Track request completion."""
        if request_id not in self.active_requests:
            return
        req_info = self.active_requests.pop(request_id)
        end_time = time.time()
        elapsed = end_time - req_info["start_time"]
        mem_end = psutil.Process().memory_info().rss / (1024 * 1024)
        mem_delta = mem_end - req_info["mem_start"]
        # Update stats
        endpoint = req_info["endpoint"]
        if endpoint in self.endpoint_stats:
            self.endpoint_stats[endpoint]["total_time"] += elapsed
            if success:
                self.endpoint_stats[endpoint]["success"] += 1
            else:
                self.endpoint_stats[endpoint]["errors"] += 1
            if pool_hit:
                self.endpoint_stats[endpoint]["pool_hits"] += 1
        # Add to completed queue
        completed = {
            **req_info,
            "end_time": end_time,
            "elapsed": round(elapsed, 2),
            "mem_delta": round(mem_delta, 1),
            "success": success,
            "error": error,
            "status_code": status_code,
            "pool_hit": pool_hit,
            "container_id": self.container_id
        }
        self.completed_requests.append(completed)
        # Persist to Redis
        await self._persist_completed_requests()
        await self._persist_active_requests()  # Update active (removed this request)
        # Track errors
        if not success and error:
            error_entry = {
                "timestamp": end_time,
                "endpoint": endpoint,
                "url": req_info["url"],
                "error": error,
                "request_id": request_id,
                "message": error,
                "level": "ERROR",
                "container_id": self.container_id
            }
            self.errors.append(error_entry)
            await self._persist_errors()
        await self._persist_endpoint_stats()
    async def track_janitor_event(self, event_type: str, sig: str, details: Dict):
        """Track janitor cleanup events."""
        self.janitor_events.append({
            "timestamp": time.time(),
            "type": event_type,  # "close_cold", "close_hot", "promote"
            "sig": sig[:8],
            "details": details,
            "container_id": self.container_id
        })
        await self._persist_janitor_events()
    def _cleanup_old_entries(self, max_age_seconds: int = 300):
        """Remove entries older than max_age_seconds (default 5min)."""
        now = time.time()
        cutoff = now - max_age_seconds
        # Clean completed requests
        while self.completed_requests and self.completed_requests[0].get("end_time", 0) < cutoff:
            self.completed_requests.popleft()
        # Clean janitor events
        while self.janitor_events and self.janitor_events[0].get("timestamp", 0) < cutoff:
            self.janitor_events.popleft()
        # Clean errors
        while self.errors and self.errors[0].get("timestamp", 0) < cutoff:
            self.errors.popleft()
    async def update_timeline(self):
        """Update timeline data points (called every 5s)."""
        now = time.time()
        mem_pct = get_container_memory_percent()
        # Clean old entries (keep last 5 minutes)
        self._cleanup_old_entries(max_age_seconds=300)
        # Count requests in last 5s
        recent_reqs = sum(1 for req in self.completed_requests
                         if now - req.get("end_time", 0) < 5)
        # Browser counts (acquire lock with timeout to prevent deadlock)
        from crawler_pool import PERMANENT, HOT_POOL, COLD_POOL, LOCK
        try:
            async with asyncio.timeout(2.0):
                async with LOCK:
                    browser_count = {
                        "permanent": 1 if PERMANENT else 0,
                        "hot": len(HOT_POOL),
                        "cold": len(COLD_POOL)
                    }
        except asyncio.TimeoutError:
            logger.warning("Lock acquisition timeout in update_timeline, using cached browser counts")
            # Use last known values or defaults
            browser_count = {
                "permanent": 1,
                "hot": 0,
                "cold": 0
            }
        self.memory_timeline.append({"time": now, "value": mem_pct})
        self.requests_timeline.append({"time": now, "value": recent_reqs})
        self.browser_timeline.append({"time": now, "browsers": browser_count})
    async def _persist_endpoint_stats(self):
        """Persist endpoint stats to Redis with retry logic."""
        max_retries = 3
        for attempt in range(max_retries):
            try:
                await self.redis.set(
                    "monitor:endpoint_stats",
                    json.dumps(self.endpoint_stats),
                    ex=self.ttl.endpoint_stats
                )
                return  # Success
            except aioredis.ConnectionError as e:
                if attempt < max_retries - 1:
                    backoff = 0.5 * (2 ** attempt)  # 0.5s, 1s, 2s
                    logger.warning(f"Redis connection error persisting endpoint stats (attempt {attempt + 1}/{max_retries}), retrying in {backoff}s: {e}")
                    await asyncio.sleep(backoff)
                else:
                    logger.error(f"Failed to persist endpoint stats after {max_retries} attempts: {e}")
            except Exception as e:
                logger.error(f"Non-retryable error persisting endpoint stats: {e}")
                break
    async def _persist_active_requests(self):
        """Persist active requests to Redis with retry logic."""
        max_retries = 3
        for attempt in range(max_retries):
            try:
                if self.active_requests:
                    await self.redis.set(
                        f"monitor:{self.container_id}:active_requests",
                        json.dumps(list(self.active_requests.values())),
                        ex=self.ttl.active_requests
                    )
                else:
                    await self.redis.delete(f"monitor:{self.container_id}:active_requests")
                return  # Success
            except aioredis.ConnectionError as e:
                if attempt < max_retries - 1:
                    backoff = 0.5 * (2 ** attempt)  # 0.5s, 1s, 2s
                    logger.warning(f"Redis connection error persisting active requests (attempt {attempt + 1}/{max_retries}), retrying in {backoff}s: {e}")
                    await asyncio.sleep(backoff)
                else:
                    logger.error(f"Failed to persist active requests after {max_retries} attempts: {e}")
            except Exception as e:
                logger.error(f"Non-retryable error persisting active requests: {e}")
                break
    async def _persist_completed_requests(self):
        """Persist completed requests to Redis with retry logic."""
        max_retries = 3
        for attempt in range(max_retries):
            try:
                await self.redis.set(
                    f"monitor:{self.container_id}:completed",
                    json.dumps(list(self.completed_requests)),
                    ex=self.ttl.completed_requests
                )
                return  # Success
            except aioredis.ConnectionError as e:
                if attempt < max_retries - 1:
                    backoff = 0.5 * (2 ** attempt)  # 0.5s, 1s, 2s
                    logger.warning(f"Redis connection error persisting completed requests (attempt {attempt + 1}/{max_retries}), retrying in {backoff}s: {e}")
                    await asyncio.sleep(backoff)
                else:
                    logger.error(f"Failed to persist completed requests after {max_retries} attempts: {e}")
            except Exception as e:
                logger.error(f"Non-retryable error persisting completed requests: {e}")
                break
    async def _persist_janitor_events(self):
        """Persist janitor events to Redis with retry logic."""
        max_retries = 3
        for attempt in range(max_retries):
            try:
                await self.redis.set(
                    f"monitor:{self.container_id}:janitor",
                    json.dumps(list(self.janitor_events)),
                    ex=self.ttl.janitor_events
                )
                return  # Success
            except aioredis.ConnectionError as e:
                if attempt < max_retries - 1:
                    backoff = 0.5 * (2 ** attempt)  # 0.5s, 1s, 2s
                    logger.warning(f"Redis connection error persisting janitor events (attempt {attempt + 1}/{max_retries}), retrying in {backoff}s: {e}")
                    await asyncio.sleep(backoff)
                else:
                    logger.error(f"Failed to persist janitor events after {max_retries} attempts: {e}")
            except Exception as e:
                logger.error(f"Non-retryable error persisting janitor events: {e}")
                break
    async def _persist_errors(self):
        """Persist errors to Redis with retry logic."""
        max_retries = 3
        for attempt in range(max_retries):
            try:
                await self.redis.set(
                    f"monitor:{self.container_id}:errors",
                    json.dumps(list(self.errors)),
                    ex=self.ttl.errors
                )
                return  # Success
            except aioredis.ConnectionError as e:
                if attempt < max_retries - 1:
                    backoff = 0.5 * (2 ** attempt)  # 0.5s, 1s, 2s
                    logger.warning(f"Redis connection error persisting errors (attempt {attempt + 1}/{max_retries}), retrying in {backoff}s: {e}")
                    await asyncio.sleep(backoff)
                else:
                    logger.error(f"Failed to persist errors after {max_retries} attempts: {e}")
            except Exception as e:
                logger.error(f"Non-retryable error persisting errors: {e}")
                break
    async def _persistence_worker(self):
        """Background worker to persist stats to Redis."""
        while True:
            try:
                await self._persist_queue.get()
                await self._persist_endpoint_stats()
                self._persist_queue.task_done()
            except asyncio.CancelledError:
                break
            except Exception as e:
                logger.error(f"Persistence worker error: {e}")
    def start_persistence_worker(self):
        """Start the background persistence worker."""
        if not self._persist_worker_task:
            self._persist_worker_task = asyncio.create_task(self._persistence_worker())
            logger.info("Started persistence worker")
    async def stop_persistence_worker(self):
        """Stop the background persistence worker."""
        if self._persist_worker_task:
            self._persist_worker_task.cancel()
            try:
                await self._persist_worker_task
            except asyncio.CancelledError:
                pass
            self._persist_worker_task = None
            logger.info("Stopped persistence worker")
    async def _heartbeat_worker(self):
        """Send heartbeat to Redis every 30s with circuit breaker for failures."""
        from utils import detect_deployment_mode
        import os
        heartbeat_failures = 0
        max_failures = 5  # Circuit breaker threshold
        while True:
            try:
                # Get hostname/container name for friendly display
                # Try HOSTNAME env var first (set by Docker Compose), then socket.gethostname()
                import socket
                hostname = os.getenv("HOSTNAME", socket.gethostname())
                # Register this container
                mode, containers = detect_deployment_mode()
                container_info = {
                    "id": self.container_id,
                    "hostname": hostname,
                    "last_seen": time.time(),
                    "mode": mode,
                    "failure_count": heartbeat_failures
                }
                # Set heartbeat with configured TTL
                await self.redis.setex(
                    f"monitor:heartbeat:{self.container_id}",
                    self.ttl.heartbeat,
                    json.dumps(container_info)
                )
                # Add to active containers set
                await self.redis.sadd("monitor:active_containers", self.container_id)
                # Reset failure counter on success
                heartbeat_failures = 0
                # Wait 30s before next heartbeat
                await asyncio.sleep(30)
            except asyncio.CancelledError:
                break
            except aioredis.ConnectionError as e:
                heartbeat_failures += 1
                logger.error(
                    f"Heartbeat Redis connection error (attempt {heartbeat_failures}/{max_failures}): {e}"
                )
                if heartbeat_failures >= max_failures:
                    # Circuit breaker - back off for longer
                    logger.critical(
                        f"Heartbeat circuit breaker triggered after {heartbeat_failures} failures. "
                        f"Container will appear offline for 5 minutes."
                    )
                    await asyncio.sleep(300)  # 5 min backoff
                    heartbeat_failures = 0
                else:
                    # Exponential backoff
                    backoff = min(30 * (2 ** heartbeat_failures), 300)
                    await asyncio.sleep(backoff)
            except Exception as e:
                logger.error(f"Unexpected heartbeat error: {e}", exc_info=True)
                await asyncio.sleep(30)
    def start_heartbeat(self):
        """Start the heartbeat worker."""
        if not self._heartbeat_task:
            self._heartbeat_task = asyncio.create_task(self._heartbeat_worker())
            logger.info("Started heartbeat worker")
    async def stop_heartbeat(self):
        """Stop the heartbeat worker and immediately deregister container."""
        if self._heartbeat_task:
            self._heartbeat_task.cancel()
            try:
                await self._heartbeat_task
            except asyncio.CancelledError:
                pass
            # Immediate deregistration (no 60s wait)
            try:
                await self.redis.srem("monitor:active_containers", self.container_id)
                await self.redis.delete(f"monitor:heartbeat:{self.container_id}")
                logger.info(f"Container {self.container_id} immediately deregistered from monitoring")
            except Exception as e:
                logger.warning(f"Failed to deregister container on shutdown: {e}")
            self._heartbeat_task = None
            logger.info("Stopped heartbeat worker")
    async def cleanup(self):
        """Cleanup on shutdown - persist final stats and stop workers."""
        logger.info("Monitor cleanup starting...")
        try:
            # Persist final stats before shutdown
            await self._persist_endpoint_stats()
            # Stop background workers
            await self.stop_persistence_worker()
            await self.stop_heartbeat()
            logger.info("Monitor cleanup completed")
        except Exception as e:
            logger.error(f"Monitor cleanup error: {e}")
    async def load_from_redis(self):
        """Load persisted stats from Redis and start workers."""
        try:
            data = await self.redis.get("monitor:endpoint_stats")
            if data:
                self.endpoint_stats = json.loads(data)
                logger.info("Loaded endpoint stats from Redis")
            # Start background workers
            self.start_heartbeat()
        except Exception as e:
            logger.warning(f"Failed to load from Redis: {e}")
    async def get_health_summary(self) -> Dict:
        """Get current system health snapshot."""
        mem_pct = get_container_memory_percent()
        cpu_pct = psutil.cpu_percent(interval=0.1)
        # Network I/O (delta since last call)
        net = psutil.net_io_counters()
        # Pool status (acquire lock with timeout to prevent race conditions)
        from crawler_pool import PERMANENT, HOT_POOL, COLD_POOL, LOCK
        try:
            async with asyncio.timeout(2.0):
                async with LOCK:
                    # TODO: Track actual browser process memory instead of estimates
                    # These are conservative estimates based on typical Chromium usage
                    permanent_mem = 270 if PERMANENT else 0  # Estimate: ~270MB for permanent browser
                    hot_mem = len(HOT_POOL) * 180  # Estimate: ~180MB per hot pool browser
                    cold_mem = len(COLD_POOL) * 180  # Estimate: ~180MB per cold pool browser
                    permanent_active = PERMANENT is not None
                    hot_count = len(HOT_POOL)
                    cold_count = len(COLD_POOL)
        except asyncio.TimeoutError:
            logger.warning("Lock acquisition timeout in get_health_summary, using defaults")
            # Use safe defaults when lock times out
            permanent_mem = 0
            hot_mem = 0
            cold_mem = 0
            permanent_active = False
            hot_count = 0
            cold_count = 0
        return {
            "container": {
                "memory_percent": round(mem_pct, 1),
                "cpu_percent": round(cpu_pct, 1),
                "network_sent_mb": round(net.bytes_sent / (1024**2), 2),
                "network_recv_mb": round(net.bytes_recv / (1024**2), 2),
                "uptime_seconds": int(time.time() - self.start_time)
            },
            "pool": {
                "permanent": {"active": permanent_active, "memory_mb": permanent_mem},
                "hot": {"count": hot_count, "memory_mb": hot_mem},
                "cold": {"count": cold_count, "memory_mb": cold_mem},
                "total_memory_mb": permanent_mem + hot_mem + cold_mem
            },
            "janitor": {
                "next_cleanup_estimate": "adaptive",  # Would need janitor state
                "memory_pressure": "LOW" if mem_pct < 60 else "MEDIUM" if mem_pct < 80 else "HIGH"
            }
        }
    def get_active_requests(self) -> List[Dict]:
        """Get list of currently active requests."""
        now = time.time()
        return [
            {
                **req,
                "elapsed": round(now - req["start_time"], 1),
                "status": "running"
            }
            for req in self.active_requests.values()
        ]
    def get_completed_requests(self, limit: int = 50, filter_status: str = "all") -> List[Dict]:
        """Get recent completed requests."""
        requests = list(self.completed_requests)[-limit:]
        if filter_status == "success":
            requests = [r for r in requests if r.get("success")]
        elif filter_status == "error":
            requests = [r for r in requests if not r.get("success")]
        return requests
    async def get_browser_list(self) -> List[Dict]:
        """Get detailed browser pool information with timeout protection."""
        from crawler_pool import PERMANENT, HOT_POOL, COLD_POOL, LAST_USED, USAGE_COUNT, DEFAULT_CONFIG_SIG, LOCK
        browsers = []
        now = time.time()
        # Acquire lock with timeout to prevent deadlock
        try:
            async with asyncio.timeout(2.0):
                async with LOCK:
                    if PERMANENT:
                        browsers.append({
                            "type": "permanent",
                            "sig": DEFAULT_CONFIG_SIG[:8] if DEFAULT_CONFIG_SIG else "unknown",
                            "age_seconds": int(now - self.start_time),
                            "last_used_seconds": int(now - LAST_USED.get(DEFAULT_CONFIG_SIG, now)),
                            "memory_mb": 270,
                            "hits": USAGE_COUNT.get(DEFAULT_CONFIG_SIG, 0),
                            "killable": False
                        })
                    for sig, crawler in HOT_POOL.items():
                        browsers.append({
                            "type": "hot",
                            "sig": sig[:8],
                            "age_seconds": int(now - self.start_time),  # Approximation
                            "last_used_seconds": int(now - LAST_USED.get(sig, now)),
                            "memory_mb": 180,  # Estimate
                            "hits": USAGE_COUNT.get(sig, 0),
                            "killable": True
                        })
                    for sig, crawler in COLD_POOL.items():
                        browsers.append({
                            "type": "cold",
                            "sig": sig[:8],
                            "age_seconds": int(now - self.start_time),
                            "last_used_seconds": int(now - LAST_USED.get(sig, now)),
                            "memory_mb": 180,
                            "hits": USAGE_COUNT.get(sig, 0),
                            "killable": True
                        })
        except asyncio.TimeoutError:
            logger.error("Browser list lock timeout - pool may be locked by janitor")
            # Return empty list when lock times out to prevent blocking
            return []
        return browsers
    def get_endpoint_stats_summary(self) -> Dict[str, Dict]:
        """Get aggregated endpoint statistics."""
        summary = {}
        for endpoint, stats in self.endpoint_stats.items():
            count = stats["count"]
            avg_time = (stats["total_time"] / count) if count > 0 else 0
            success_rate = (stats["success"] / count * 100) if count > 0 else 0
            pool_hit_rate = (stats["pool_hits"] / count * 100) if count > 0 else 0
            summary[endpoint] = {
                "count": count,
                "avg_latency_ms": round(avg_time * 1000, 1),
                "success_rate_percent": round(success_rate, 1),
                "pool_hit_rate_percent": round(pool_hit_rate, 1),
                "errors": stats["errors"]
            }
        return summary
    def get_timeline_data(self, metric: str, window: str = "5m") -> Dict:
        """Get timeline data for charts."""
        # For now, only 5m window supported
        if metric == "memory":
            data = list(self.memory_timeline)
        elif metric == "requests":
            data = list(self.requests_timeline)
        elif metric == "browsers":
            data = list(self.browser_timeline)
        else:
            return {"timestamps": [], "values": []}
        return {
            "timestamps": [int(d["time"]) for d in data],
            "values": [d.get("value", d.get("browsers")) for d in data]
        }
    def get_janitor_log(self, limit: int = 100) -> List[Dict]:
        """Get recent janitor events."""
        return list(self.janitor_events)[-limit:]
    def get_errors_log(self, limit: int = 100) -> List[Dict]:
        """Get recent errors."""
        return list(self.errors)[-limit:]
 # Global instance (initialized in server.py)
 monitor_stats: Optional[MonitorStats] = None
 def get_monitor() -> MonitorStats:
    """Get global monitor instance."""
    if monitor_stats is None:
        raise RuntimeError("Monitor not initialized")
    return monitor_stats
--- a/deploy/docker/monitor_routes.py
+++ b/deploy/docker/monitor_routes.py
@@ -1,608 +0,0 @@
 # monitor_routes.py - Monitor API endpoints
 from fastapi import APIRouter, HTTPException, WebSocket, WebSocketDisconnect
 from pydantic import BaseModel
 from typing import Optional
 from monitor import get_monitor
 from utils import detect_deployment_mode, get_container_id
 import logging
 import asyncio
 import json
 import re
 logger = logging.getLogger(__name__)
 router = APIRouter(prefix="/monitor", tags=["monitor"])
 # ========== Security & Validation ==========
 def validate_container_id(cid: str) -> bool:
    """Validate container ID format to prevent Redis key injection.
    Docker container IDs are 12-64 character hexadecimal strings.
    Hostnames are alphanumeric with dashes and underscores.
    Args:
        cid: Container ID to validate
    Returns:
        True if valid, False otherwise
    """
    if not cid or not isinstance(cid, str):
        return False
    # Allow alphanumeric, dashes, and underscores only (1-64 chars)
    # This prevents path traversal (../../), wildcards (**), and other injection attempts
    return bool(re.match(r'^[a-zA-Z0-9_-]{1,64}$', cid))
 # ========== Redis Aggregation Helpers ==========
 async def _get_active_containers():
    """Get list of active container IDs from Redis with validation."""
    try:
        monitor = get_monitor()
        container_ids = await monitor.redis.smembers("monitor:active_containers")
        # Decode and validate each container ID
        validated = []
        for cid in container_ids:
            cid_str = cid.decode() if isinstance(cid, bytes) else cid
            if validate_container_id(cid_str):
                validated.append(cid_str)
            else:
                logger.warning(f"Invalid container ID format rejected: {cid_str}")
        return validated
    except Exception as e:
        logger.error(f"Failed to get active containers: {e}")
        return []
 async def _aggregate_active_requests():
    """Aggregate active requests from all containers."""
    container_ids = await _get_active_containers()
    all_requests = []
    monitor = get_monitor()
    for container_id in container_ids:
        try:
            data = await monitor.redis.get(f"monitor:{container_id}:active_requests")
            if data:
                requests = json.loads(data)
                all_requests.extend(requests)
        except Exception as e:
            logger.warning(f"Failed to get active requests from {container_id}: {e}")
    return all_requests
 async def _aggregate_completed_requests(limit=100):
    """Aggregate completed requests from all containers."""
    container_ids = await _get_active_containers()
    all_requests = []
    monitor = get_monitor()
    for container_id in container_ids:
        try:
            data = await monitor.redis.get(f"monitor:{container_id}:completed")
            if data:
                requests = json.loads(data)
                all_requests.extend(requests)
        except Exception as e:
            logger.warning(f"Failed to get completed requests from {container_id}: {e}")
    # Sort by end_time (most recent first) and limit
    all_requests.sort(key=lambda x: x.get("end_time", 0), reverse=True)
    return all_requests[:limit]
 async def _aggregate_janitor_events(limit=100):
    """Aggregate janitor events from all containers."""
    container_ids = await _get_active_containers()
    all_events = []
    monitor = get_monitor()
    for container_id in container_ids:
        try:
            data = await monitor.redis.get(f"monitor:{container_id}:janitor")
            if data:
                events = json.loads(data)
                all_events.extend(events)
        except Exception as e:
            logger.warning(f"Failed to get janitor events from {container_id}: {e}")
    # Sort by timestamp (most recent first) and limit
    all_events.sort(key=lambda x: x.get("timestamp", 0), reverse=True)
    return all_events[:limit]
 async def _aggregate_errors(limit=100):
    """Aggregate errors from all containers."""
    container_ids = await _get_active_containers()
    all_errors = []
    monitor = get_monitor()
    for container_id in container_ids:
        try:
            data = await monitor.redis.get(f"monitor:{container_id}:errors")
            if data:
                errors = json.loads(data)
                all_errors.extend(errors)
        except Exception as e:
            logger.warning(f"Failed to get errors from {container_id}: {e}")
    # Sort by timestamp (most recent first) and limit
    all_errors.sort(key=lambda x: x.get("timestamp", 0), reverse=True)
    return all_errors[:limit]
@router.get("/health")
 async def get_health():
    """Get current system health snapshot."""
    try:
        monitor = get_monitor()
        return await monitor.get_health_summary()
    except Exception as e:
        logger.error(f"Error getting health: {e}")
        raise HTTPException(500, str(e))
@router.get("/requests")
 async def get_requests(status: str = "all", limit: int = 50):
    """Get active and completed requests.
    Args:
        status: Filter by 'active', 'completed', 'success', 'error', or 'all'
        limit: Max number of completed requests to return (default 50)
    """
    # Input validation
    if status not in ["all", "active", "completed", "success", "error"]:
        raise HTTPException(400, f"Invalid status: {status}. Must be one of: all, active, completed, success, error")
    if limit < 1 or limit > 1000:
        raise HTTPException(400, f"Invalid limit: {limit}. Must be between 1 and 1000")
    try:
        # Aggregate from all containers via Redis
        active_requests = await _aggregate_active_requests()
        completed_requests = await _aggregate_completed_requests(limit)
        # Filter by status if needed
        if status in ["success", "error"]:
            is_success = (status == "success")
            completed_requests = [r for r in completed_requests if r.get("success") == is_success]
        if status == "active":
            return {"active": active_requests, "completed": []}
        elif status == "completed":
            return {"active": [], "completed": completed_requests}
        else:  # "all" or success/error
            return {
                "active": active_requests,
                "completed": completed_requests
            }
    except Exception as e:
        logger.error(f"Error getting requests: {e}")
        raise HTTPException(500, str(e))
@router.get("/browsers")
 async def get_browsers():
    """Get detailed browser pool information."""
    try:
        monitor = get_monitor()
        container_id = get_container_id()
        browsers = await monitor.get_browser_list()
        # Add container_id to each browser
        for browser in browsers:
            browser["container_id"] = container_id
        # Calculate summary stats
        total_browsers = len(browsers)
        total_memory = sum(b["memory_mb"] for b in browsers)
        # Calculate reuse rate from recent requests
        recent = monitor.get_completed_requests(100)
        pool_hits = sum(1 for r in recent if r.get("pool_hit", False))
        reuse_rate = (pool_hits / len(recent) * 100) if recent else 0
        return {
            "browsers": browsers,
            "summary": {
                "total_count": total_browsers,
                "total_memory_mb": total_memory,
                "reuse_rate_percent": round(reuse_rate, 1)
            },
            "container_id": container_id
        }
    except Exception as e:
        logger.error(f"Error getting browsers: {e}")
        raise HTTPException(500, str(e))
@router.get("/endpoints/stats")
 async def get_endpoint_stats():
    """Get aggregated endpoint statistics."""
    try:
        monitor = get_monitor()
        return monitor.get_endpoint_stats_summary()
    except Exception as e:
        logger.error(f"Error getting endpoint stats: {e}")
        raise HTTPException(500, str(e))
@router.get("/timeline")
 async def get_timeline(metric: str = "memory", window: str = "5m"):
    """Get timeline data for charts.
    Args:
        metric: 'memory', 'requests', or 'browsers'
        window: Time window (only '5m' supported for now)
    """
    # Input validation
    if metric not in ["memory", "requests", "browsers"]:
        raise HTTPException(400, f"Invalid metric: {metric}. Must be one of: memory, requests, browsers")
    if window != "5m":
        raise HTTPException(400, f"Invalid window: {window}. Only '5m' is currently supported")
    try:
        monitor = get_monitor()
        return monitor.get_timeline_data(metric, window)
    except Exception as e:
        logger.error(f"Error getting timeline: {e}")
        raise HTTPException(500, str(e))
@router.get("/logs/janitor")
 async def get_janitor_log(limit: int = 100):
    """Get recent janitor cleanup events."""
    # Input validation
    if limit < 1 or limit > 1000:
        raise HTTPException(400, f"Invalid limit: {limit}. Must be between 1 and 1000")
    try:
        # Aggregate from all containers via Redis
        events = await _aggregate_janitor_events(limit)
        return {"events": events}
    except Exception as e:
        logger.error(f"Error getting janitor log: {e}")
        raise HTTPException(500, str(e))
@router.get("/logs/errors")
 async def get_errors_log(limit: int = 100):
    """Get recent errors."""
    # Input validation
    if limit < 1 or limit > 1000:
        raise HTTPException(400, f"Invalid limit: {limit}. Must be between 1 and 1000")
    try:
        # Aggregate from all containers via Redis
        errors = await _aggregate_errors(limit)
        return {"errors": errors}
    except Exception as e:
        logger.error(f"Error getting errors log: {e}")
        raise HTTPException(500, str(e))
 # ========== Control Actions ==========
 class KillBrowserRequest(BaseModel):
    sig: str
@router.post("/actions/cleanup")
 async def force_cleanup():
    """Force immediate janitor cleanup (kills idle cold pool browsers)."""
    try:
        from crawler_pool import COLD_POOL, LAST_USED, USAGE_COUNT, LOCK
        import time
        from contextlib import suppress
        killed_count = 0
        now = time.time()
        async with LOCK:
            for sig in list(COLD_POOL.keys()):
                # Kill all cold pool browsers immediately
                logger.info(f"🧹 Force cleanup: closing cold browser (sig={sig[:8]})")
                with suppress(Exception):
                    await COLD_POOL[sig].close()
                COLD_POOL.pop(sig, None)
                LAST_USED.pop(sig, None)
                USAGE_COUNT.pop(sig, None)
                killed_count += 1
        monitor = get_monitor()
        await monitor.track_janitor_event("force_cleanup", "manual", {"killed": killed_count})
        return {"success": True, "killed_browsers": killed_count}
    except Exception as e:
        logger.error(f"Error during force cleanup: {e}")
        raise HTTPException(500, str(e))
@router.post("/actions/kill_browser")
 async def kill_browser(req: KillBrowserRequest):
    """Kill a specific browser by signature (hot or cold only).
    Args:
        sig: Browser config signature (first 8 chars)
    """
    try:
        from crawler_pool import HOT_POOL, COLD_POOL, LAST_USED, USAGE_COUNT, LOCK, DEFAULT_CONFIG_SIG
        from contextlib import suppress
        # Find full signature matching prefix
        target_sig = None
        pool_type = None
        async with LOCK:
            # Check hot pool
            for sig in HOT_POOL.keys():
                if sig.startswith(req.sig):
                    target_sig = sig
                    pool_type = "hot"
                    break
            # Check cold pool
            if not target_sig:
                for sig in COLD_POOL.keys():
                    if sig.startswith(req.sig):
                        target_sig = sig
                        pool_type = "cold"
                        break
            # Check if trying to kill permanent
            if DEFAULT_CONFIG_SIG and DEFAULT_CONFIG_SIG.startswith(req.sig):
                raise HTTPException(403, "Cannot kill permanent browser. Use restart instead.")
            if not target_sig:
                raise HTTPException(404, f"Browser with sig={req.sig} not found")
            # Warn if there are active requests (browser might be in use)
            monitor = get_monitor()
            active_count = len(monitor.get_active_requests())
            if active_count > 0:
                logger.warning(f"Killing browser {target_sig[:8]} while {active_count} requests are active - may cause failures")
            # Kill the browser
            if pool_type == "hot":
                browser = HOT_POOL.pop(target_sig)
            else:
                browser = COLD_POOL.pop(target_sig)
            with suppress(Exception):
                await browser.close()
            LAST_USED.pop(target_sig, None)
            USAGE_COUNT.pop(target_sig, None)
        logger.info(f"🔪 Killed {pool_type} browser (sig={target_sig[:8]})")
        monitor = get_monitor()
        await monitor.track_janitor_event("kill_browser", target_sig, {"pool": pool_type, "manual": True})
        return {"success": True, "killed_sig": target_sig[:8], "pool_type": pool_type}
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error killing browser: {e}")
        raise HTTPException(500, str(e))
@router.post("/actions/restart_browser")
 async def restart_browser(req: KillBrowserRequest):
    """Restart a browser (kill + recreate). Works for permanent too.
    Args:
        sig: Browser config signature (first 8 chars), or "permanent"
    """
    try:
        from crawler_pool import (PERMANENT, HOT_POOL, COLD_POOL, LAST_USED,
                                  USAGE_COUNT, LOCK, DEFAULT_CONFIG_SIG, init_permanent)
        from crawl4ai import AsyncWebCrawler, BrowserConfig
        from contextlib import suppress
        import time
        # Handle permanent browser restart
        if req.sig == "permanent" or (DEFAULT_CONFIG_SIG and DEFAULT_CONFIG_SIG.startswith(req.sig)):
            async with LOCK:
                if PERMANENT:
                    with suppress(Exception):
                        await PERMANENT.close()
                # Reinitialize permanent
                from utils import load_config
                config = load_config()
                await init_permanent(BrowserConfig(
                    extra_args=config["crawler"]["browser"].get("extra_args", []),
                    **config["crawler"]["browser"].get("kwargs", {}),
                ))
            logger.info("🔄 Restarted permanent browser")
            return {"success": True, "restarted": "permanent"}
        # Handle hot/cold browser restart
        target_sig = None
        pool_type = None
        browser_config = None
        async with LOCK:
            # Find browser
            for sig in HOT_POOL.keys():
                if sig.startswith(req.sig):
                    target_sig = sig
                    pool_type = "hot"
                    # Would need to reconstruct config (not stored currently)
                    break
            if not target_sig:
                for sig in COLD_POOL.keys():
                    if sig.startswith(req.sig):
                        target_sig = sig
                        pool_type = "cold"
                        break
            if not target_sig:
                raise HTTPException(404, f"Browser with sig={req.sig} not found")
            # Kill existing
            if pool_type == "hot":
                browser = HOT_POOL.pop(target_sig)
            else:
                browser = COLD_POOL.pop(target_sig)
            with suppress(Exception):
                await browser.close()
            # Note: We can't easily recreate with same config without storing it
            # For now, just kill and let new requests create fresh ones
            LAST_USED.pop(target_sig, None)
            USAGE_COUNT.pop(target_sig, None)
        logger.info(f"🔄 Restarted {pool_type} browser (sig={target_sig[:8]})")
        monitor = get_monitor()
        await monitor.track_janitor_event("restart_browser", target_sig, {"pool": pool_type})
        return {"success": True, "restarted_sig": target_sig[:8], "note": "Browser will be recreated on next request"}
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error restarting browser: {e}")
        raise HTTPException(500, str(e))
@router.post("/stats/reset")
 async def reset_stats():
    """Reset today's endpoint counters."""
    try:
        monitor = get_monitor()
        monitor.endpoint_stats.clear()
        await monitor._persist_endpoint_stats()
        return {"success": True, "message": "Endpoint stats reset"}
    except Exception as e:
        logger.error(f"Error resetting stats: {e}")
        raise HTTPException(500, str(e))
@router.get("/containers")
 async def get_containers():
    """Get container deployment info from Redis heartbeats."""
    try:
        monitor = get_monitor()
        container_ids = await _get_active_containers()
        containers = []
        for cid in container_ids:
            try:
                # Get heartbeat data
                data = await monitor.redis.get(f"monitor:heartbeat:{cid}")
                if data:
                    info = json.loads(data)
                    containers.append({
                        "id": info.get("id", cid),
                        "hostname": info.get("hostname", cid),
                        "healthy": True  # If heartbeat exists, it's healthy
                    })
            except Exception as e:
                logger.warning(f"Failed to get heartbeat for {cid}: {e}")
        # Determine mode
        mode = "single" if len(containers) == 1 else "compose"
        if len(containers) > 1:
            # Check if any hostname has swarm pattern (service.slot.task_id)
            if any("." in c["hostname"] and len(c["hostname"].split(".")) > 2 for c in containers):
                mode = "swarm"
        return {
            "mode": mode,
            "container_id": get_container_id(),
            "containers": containers,
            "count": len(containers)
        }
    except Exception as e:
        logger.error(f"Error getting containers: {e}")
        raise HTTPException(500, str(e))
@router.websocket("/ws")
 async def websocket_endpoint(websocket: WebSocket):
    """WebSocket endpoint for real-time monitoring updates.
    Sends aggregated updates every 2 seconds from all containers with:
    - Health stats (local container)
    - Active/completed requests (aggregated from all containers)
    - Browser pool status (local container only - not in Redis)
    - Timeline data (local container - TODO: aggregate from Redis)
    - Janitor events (aggregated from all containers)
    - Errors (aggregated from all containers)
    """
    await websocket.accept()
    logger.info("WebSocket client connected")
    try:
        while True:
            try:
                # Gather aggregated monitoring data from Redis
                monitor = get_monitor()
                container_id = get_container_id()
                # Get container info
                containers_info = await get_containers()
                # AGGREGATE data from all containers via Redis
                active_reqs = await _aggregate_active_requests()
                completed_reqs = await _aggregate_completed_requests(limit=10)
                janitor_events = await _aggregate_janitor_events(limit=10)
                errors_log = await _aggregate_errors(limit=10)
                # Local container data (not aggregated)
                local_health = await monitor.get_health_summary()
                browsers = await monitor.get_browser_list()  # Browser list is local only
                # Add container_id to browsers (they're local)
                for browser in browsers:
                    browser["container_id"] = container_id
                data = {
                    "timestamp": asyncio.get_event_loop().time(),
                    "container_id": container_id,  # This container handling the WebSocket
                    "is_aggregated": True,  # Flag to indicate aggregated data
                    "local_health": local_health,  # This container's health
                    "containers": containers_info.get("containers", []),  # All containers
                    "requests": {
                        "active": active_reqs,  # Aggregated from all containers
                        "completed": completed_reqs  # Aggregated from all containers
                    },
                    "browsers": browsers,  # Local only (not in Redis)
                    "timeline": {
                        # TODO: Aggregate timeline from Redis (currently local only)
                        "memory": monitor.get_timeline_data("memory", "5m"),
                        "requests": monitor.get_timeline_data("requests", "5m"),
                        "browsers": monitor.get_timeline_data("browsers", "5m")
                    },
                    "janitor": janitor_events,  # Aggregated from all containers
                    "errors": errors_log  # Aggregated from all containers
                }
                # Send update to client
                await websocket.send_json(data)
                # Wait 2 seconds before next update
                await asyncio.sleep(2)
            except WebSocketDisconnect:
                logger.info("WebSocket client disconnected")
                break
            except Exception as e:
                logger.error(f"WebSocket error: {e}", exc_info=True)
                await asyncio.sleep(2)  # Continue trying
    except Exception as e:
        logger.error(f"WebSocket connection error: {e}", exc_info=True)
    finally:
        logger.info("WebSocket connection closed")
--- a/deploy/docker/server.py
+++ b/deploy/docker/server.py
@@ -16,7 +16,6 @@ from fastapi import Request, Depends
 from fastapi.responses import FileResponse
 import base64
 import re
 import logging
 from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
 from api import (
    handle_markdown_request, handle_llm_qa,
@@ -79,14 +78,6 @@ __version__ = "0.5.1-d1"
 MAX_PAGES = config["crawler"]["pool"].get("max_pages", 30)
 GLOBAL_SEM = asyncio.Semaphore(MAX_PAGES)
 # ── default browser config helper ─────────────────────────────
 def get_default_browser_config() -> BrowserConfig:
    """Get default BrowserConfig from config.yml."""
    return BrowserConfig(
        extra_args=config["crawler"]["browser"].get("extra_args", []),
        **config["crawler"]["browser"].get("kwargs", {}),
    )
 # import logging
 # page_log = logging.getLogger("page_cap")
 # orig_arun = AsyncWebCrawler.arun
@@ -112,52 +103,15 @@ AsyncWebCrawler.arun = capped_arun
@asynccontextmanager
 async def lifespan(_: FastAPI):
-    from crawler_pool import init_permanent
+    await get_crawler(BrowserConfig(
    from monitor import MonitorStats
    import monitor as monitor_module
    # Initialize monitor
    monitor_module.monitor_stats = MonitorStats(redis)
    await monitor_module.monitor_stats.load_from_redis()
    monitor_module.monitor_stats.start_persistence_worker()
    # Initialize browser pool
    await init_permanent(BrowserConfig(
        extra_args=config["crawler"]["browser"].get("extra_args", []),
        **config["crawler"]["browser"].get("kwargs", {}),
-    ))
+    ))           # warm‑up
-
+    app.state.janitor = asyncio.create_task(janitor())        # idle GC
    # Start background tasks
    app.state.janitor = asyncio.create_task(janitor())
    app.state.timeline_updater = asyncio.create_task(_timeline_updater())
    yield
    # Cleanup
    app.state.janitor.cancel()
    app.state.timeline_updater.cancel()
    # Monitor cleanup (persist stats and stop workers)
    from monitor import get_monitor
    try:
        await get_monitor().cleanup()
    except Exception as e:
        logger.error(f"Monitor cleanup failed: {e}")
    await close_all()
 async def _timeline_updater():
    """Update timeline data every 5 seconds."""
    from monitor import get_monitor
    while True:
        await asyncio.sleep(5)
        try:
            await asyncio.wait_for(get_monitor().update_timeline(), timeout=4.0)
        except asyncio.TimeoutError:
            logger.warning("Timeline update timeout after 4s")
        except Exception as e:
            logger.warning(f"Timeline update error: {e}")
 # ───────────────────── FastAPI instance ──────────────────────
 app = FastAPI(
    title=config["app"]["title"],
@@ -175,36 +129,13 @@ app.mount(
    name="play",
 )
 # ── static monitor dashboard ────────────────────────────────
 MONITOR_DIR = pathlib.Path(__file__).parent / "static" / "monitor"
 if not MONITOR_DIR.exists():
    raise RuntimeError(f"Monitor assets not found at {MONITOR_DIR}")
 app.mount(
    "/dashboard",
    StaticFiles(directory=MONITOR_DIR, html=True),
    name="monitor_ui",
 )
 # ── static assets (logo, etc) ────────────────────────────────
 ASSETS_DIR = pathlib.Path(__file__).parent / "static" / "assets"
 if ASSETS_DIR.exists():
    app.mount(
        "/static/assets",
        StaticFiles(directory=ASSETS_DIR),
        name="assets",
    )
@app.get("/")
 async def root():
    return RedirectResponse("/playground")
 # ─────────────────── infra / middleware  ─────────────────────
-# Build Redis URL from environment or config
+redis = aioredis.from_url(config["redis"].get("uri", "redis://localhost"))
 redis_host = os.getenv("REDIS_HOST", config["redis"].get("host", "localhost"))
 redis_port = os.getenv("REDIS_PORT", config["redis"].get("port", 6379))
 redis_url = config["redis"].get("uri") or f"redis://{redis_host}:{redis_port}"
 redis = aioredis.from_url(redis_url)
 limiter = Limiter(
    key_func=get_remote_address,
@@ -281,12 +212,6 @@ def _safe_eval_config(expr: str) -> dict:
 # ── job router ──────────────────────────────────────────────
 app.include_router(init_job_router(redis, config, token_dep))
 # ── monitor router ──────────────────────────────────────────
 from monitor_routes import router as monitor_router
 app.include_router(monitor_router)
 logger = logging.getLogger(__name__)
 # ──────────────────────── Endpoints ──────────────────────────
@app.post("/token")
 async def get_token(req: TokenRequest):
@@ -341,20 +266,27 @@ async def generate_html(
    Crawls the URL, preprocesses the raw HTML for schema extraction, and returns the processed HTML.
    Use when you need sanitized HTML structures for building schemas or further processing.
    """
    from crawler_pool import get_crawler
    cfg = CrawlerRunConfig()
    try:
-        crawler = await get_crawler(get_default_browser_config())
+        async with AsyncWebCrawler(config=BrowserConfig()) as crawler:
-        results = await crawler.arun(url=body.url, config=cfg)
+            results = await crawler.arun(url=body.url, config=cfg)
        # Check if the crawl was successful
        if not results[0].success:
-            raise HTTPException(500, detail=results[0].error_message or "Crawl failed")
+            raise HTTPException(
-
+                status_code=500,
                detail=results[0].error_message or "Crawl failed"
            )
        raw_html = results[0].html
        from crawl4ai.utils import preprocess_html_for_schema
        processed_html = preprocess_html_for_schema(raw_html)
        return JSONResponse({"html": processed_html, "url": body.url, "success": True})
    except Exception as e:
-        raise HTTPException(500, detail=str(e))
+        # Log and raise as HTTP 500 for other exceptions
        raise HTTPException(
            status_code=500,
            detail=str(e)
        )
 # Screenshot endpoint
@@ -372,13 +304,16 @@ async def generate_screenshot(
    Use when you need an image snapshot of the rendered page. Its recommened to provide an output path to save the screenshot.
    Then in result instead of the screenshot you will get a path to the saved file.
    """
    from crawler_pool import get_crawler
    try:
-        cfg = CrawlerRunConfig(screenshot=True, screenshot_wait_for=body.screenshot_wait_for)
+        cfg = CrawlerRunConfig(
-        crawler = await get_crawler(get_default_browser_config())
+            screenshot=True, screenshot_wait_for=body.screenshot_wait_for)
-        results = await crawler.arun(url=body.url, config=cfg)
+        async with AsyncWebCrawler(config=BrowserConfig()) as crawler:
            results = await crawler.arun(url=body.url, config=cfg)
        if not results[0].success:
-            raise HTTPException(500, detail=results[0].error_message or "Crawl failed")
+            raise HTTPException(
                status_code=500,
                detail=results[0].error_message or "Crawl failed"
            )
        screenshot_data = results[0].screenshot
        if body.output_path:
            abs_path = os.path.abspath(body.output_path)
@@ -388,7 +323,10 @@ async def generate_screenshot(
            return {"success": True, "path": abs_path}
        return {"success": True, "screenshot": screenshot_data}
    except Exception as e:
-        raise HTTPException(500, detail=str(e))
+        raise HTTPException(
            status_code=500,
            detail=str(e)
        )
 # PDF endpoint
@@ -406,13 +344,15 @@ async def generate_pdf(
    Use when you need a printable or archivable snapshot of the page. It is recommended to provide an output path to save the PDF.
    Then in result instead of the PDF you will get a path to the saved file.
    """
    from crawler_pool import get_crawler
    try:
        cfg = CrawlerRunConfig(pdf=True)
-        crawler = await get_crawler(get_default_browser_config())
+        async with AsyncWebCrawler(config=BrowserConfig()) as crawler:
-        results = await crawler.arun(url=body.url, config=cfg)
+            results = await crawler.arun(url=body.url, config=cfg)
        if not results[0].success:
-            raise HTTPException(500, detail=results[0].error_message or "Crawl failed")
+            raise HTTPException(
                status_code=500,
                detail=results[0].error_message or "Crawl failed"
            )
        pdf_data = results[0].pdf
        if body.output_path:
            abs_path = os.path.abspath(body.output_path)
@@ -422,7 +362,10 @@ async def generate_pdf(
            return {"success": True, "path": abs_path}
        return {"success": True, "pdf": base64.b64encode(pdf_data).decode()}
    except Exception as e:
-        raise HTTPException(500, detail=str(e))
+        raise HTTPException(
            status_code=500,
            detail=str(e)
        )
@app.post("/execute_js")
@@ -478,17 +421,23 @@ async def execute_js(
        ```
    """
    from crawler_pool import get_crawler
    try:
        cfg = CrawlerRunConfig(js_code=body.scripts)
-        crawler = await get_crawler(get_default_browser_config())
+        async with AsyncWebCrawler(config=BrowserConfig()) as crawler:
-        results = await crawler.arun(url=body.url, config=cfg)
+            results = await crawler.arun(url=body.url, config=cfg)
        if not results[0].success:
-            raise HTTPException(500, detail=results[0].error_message or "Crawl failed")
+            raise HTTPException(
                status_code=500,
                detail=results[0].error_message or "Crawl failed"
            )
        # Return JSON-serializable dict of the first CrawlResult
        data = results[0].model_dump()
        return JSONResponse(data)
    except Exception as e:
-        raise HTTPException(500, detail=str(e))
+        raise HTTPException(
            status_code=500,
            detail=str(e)
        )
@app.get("/llm/{url:path}")
--- a/deploy/docker/server_manager.py
+++ b/deploy/docker/server_manager.py
--- a/deploy/docker/static/assets/crawl4ai-logo.jpg
+++ b/deploy/docker/static/assets/crawl4ai-logo.jpg
--- a/deploy/docker/static/assets/crawl4ai-logo.png
+++ b/deploy/docker/static/assets/crawl4ai-logo.png
--- a/deploy/docker/static/assets/logo.png
+++ b/deploy/docker/static/assets/logo.png
--- a/deploy/docker/static/monitor/index.html
+++ b/deploy/docker/static/monitor/index.html
--- a/deploy/docker/static/playground/index.html
+++ b/deploy/docker/static/playground/index.html
@@ -167,14 +167,11 @@
            </a>
        </h1>
-        <div class="ml-auto flex items-center space-x-4">
+        <div class="ml-auto flex space-x-2">
-            <a href="/dashboard" class="text-xs text-secondary hover:text-primary underline">Monitor</a>
+            <button id="play-tab"
-            <div class="flex space-x-2">
+                class="px-3 py-1 rounded-t bg-surface border border-b-0 border-border text-primary">Playground</button>
-                <button id="play-tab"
+            <button id="stress-tab" class="px-3 py-1 rounded-t border border-border hover:bg-surface">Stress
-                    class="px-3 py-1 rounded-t bg-surface border border-b-0 border-border text-primary">Playground</button>
+                Test</button>
                <button id="stress-tab" class="px-3 py-1 rounded-t border border-border hover:bg-surface">Stress
                    Test</button>
            </div>
        </div>
    </header>
--- a/deploy/docker/test-websocket.py
+++ b/deploy/docker/test-websocket.py
@@ -1,34 +0,0 @@
 #!/usr/bin/env python3
 """
 Quick WebSocket test - Connect to monitor WebSocket and print updates
 """
 import asyncio
 import websockets
 import json
 async def test_websocket():
    uri = "ws://localhost:11235/monitor/ws"
    print(f"Connecting to {uri}...")
    try:
        async with websockets.connect(uri) as websocket:
            print("✅ Connected!")
            # Receive and print 5 updates
            for i in range(5):
                message = await websocket.recv()
                data = json.loads(message)
                print(f"\n📊 Update #{i+1}:")
                print(f"  - Health: CPU {data['health']['container']['cpu_percent']}%, Memory {data['health']['container']['memory_percent']}%")
                print(f"  - Active Requests: {len(data['requests']['active'])}")
                print(f"  - Browsers: {len(data['browsers'])}")
    except Exception as e:
        print(f"❌ Error: {e}")
        return 1
    print("\n✅ WebSocket test passed!")
    return 0
 if __name__ == "__main__":
    exit(asyncio.run(test_websocket()))
--- a/deploy/docker/tests/cli/README.md
+++ b/deploy/docker/tests/cli/README.md
@@ -1,298 +0,0 @@
 # Crawl4AI CLI E2E Test Suite
 Comprehensive end-to-end tests for the `crwl server` command-line interface.
 ## Overview
 This test suite validates all aspects of the Docker server CLI including:
 - Basic operations (start, stop, status, logs)
 - Advanced features (scaling, modes, custom configurations)
 - Resource management and stress testing
 - Dashboard UI functionality
 - Edge cases and error handling
 **Total Tests:** 32
 - Basic: 8 tests
 - Advanced: 8 tests
 - Resource: 5 tests
 - Dashboard: 1 test
 - Edge Cases: 10 tests
 ## Prerequisites
 ```bash
 # Activate virtual environment
 source venv/bin/activate
 # For dashboard tests, install Playwright
 pip install playwright
 playwright install chromium
 # Ensure Docker is running
 docker ps
 ```
 ## Quick Start
 ```bash
 # Run all tests (except dashboard)
 ./run_tests.sh
 # Run specific category
 ./run_tests.sh basic
 ./run_tests.sh advanced
 ./run_tests.sh resource
 ./run_tests.sh edge
 # Run dashboard tests (slower, includes UI screenshots)
 ./run_tests.sh dashboard
 # Run specific test
 ./run_tests.sh basic 01
 ./run_tests.sh edge 05
 ```
 ## Test Categories
 ### 1. Basic Tests (`basic/`)
 Core CLI functionality tests.
 | Test | Description | Expected Result |
 |------|-------------|----------------|
 | `test_01_start_default.sh` | Start server with defaults | 1 replica on port 11235 |
 | `test_02_status.sh` | Check server status | Shows running state and details |
 | `test_03_stop.sh` | Stop server | Clean shutdown, port freed |
 | `test_04_start_custom_port.sh` | Start on port 8080 | Server on custom port |
 | `test_05_start_replicas.sh` | Start with 3 replicas | Multi-container deployment |
 | `test_06_logs.sh` | View server logs | Logs displayed correctly |
 | `test_07_restart.sh` | Restart server | Preserves configuration |
 | `test_08_cleanup.sh` | Force cleanup | All resources removed |
 ### 2. Advanced Tests (`advanced/`)
 Advanced features and configurations.
 | Test | Description | Expected Result |
 |------|-------------|----------------|
 | `test_01_scale_up.sh` | Scale 3 → 5 replicas | Live scaling without downtime |
 | `test_02_scale_down.sh` | Scale 5 → 2 replicas | Graceful container removal |
 | `test_03_mode_single.sh` | Explicit single mode | Single container deployment |
 | `test_04_mode_compose.sh` | Compose mode with Nginx | Multi-container with load balancer |
 | `test_05_custom_image.sh` | Custom image specification | Uses specified image tag |
 | `test_06_env_file.sh` | Environment file loading | Variables loaded correctly |
 | `test_07_stop_remove_volumes.sh` | Stop with volume removal | Volumes cleaned up |
 | `test_08_restart_with_scale.sh` | Restart with new replica count | Configuration updated |
 ### 3. Resource Tests (`resource/`)
 Resource monitoring and stress testing.
 | Test | Description | Expected Result |
 |------|-------------|----------------|
 | `test_01_memory_monitoring.sh` | Monitor memory usage | Stats accessible and reasonable |
 | `test_02_cpu_stress.sh` | Concurrent request load | Handles load without errors |
 | `test_03_max_replicas.sh` | 10 replicas stress test | Maximum scale works correctly |
 | `test_04_cleanup_verification.sh` | Verify resource cleanup | All Docker resources removed |
 | `test_05_long_running.sh` | 5-minute stability test | Server remains stable |
 ### 4. Dashboard Tests (`dashboard/`)
 Dashboard UI functionality with Playwright.
 | Test | Description | Expected Result |
 |------|-------------|----------------|
 | `test_01_dashboard_ui.py` | Full dashboard UI test | All UI elements functional |
 **Dashboard Test Details:**
 - Starts server with 3 replicas
 - Runs demo script to generate activity
 - Uses Playwright to:
  - Take screenshots of dashboard
  - Verify container filter buttons
  - Check WebSocket connection
  - Validate timeline charts
  - Test all dashboard sections
 **Screenshots saved to:** `dashboard/screenshots/`
 ### 5. Edge Case Tests (`edge/`)
 Error handling and validation.
 | Test | Description | Expected Result |
 |------|-------------|----------------|
 | `test_01_already_running.sh` | Start when already running | Proper error message |
 | `test_02_not_running.sh` | Operations when stopped | Appropriate errors |
 | `test_03_scale_single_mode.sh` | Scale single container | Error with guidance |
 | `test_04_invalid_port.sh` | Invalid port numbers | Validation errors |
 | `test_05_invalid_replicas.sh` | Invalid replica counts | Validation errors |
 | `test_06_missing_env_file.sh` | Non-existent env file | File not found error |
 | `test_07_port_in_use.sh` | Port already occupied | Port conflict error |
 | `test_08_state_corruption.sh` | Corrupted state file | Cleanup recovers |
 | `test_09_network_conflict.sh` | Docker network collision | Handles gracefully |
 | `test_10_rapid_operations.sh` | Rapid start/stop cycles | No corruption |
 ## Test Execution Workflow
 Each test follows this pattern:
 1. **Setup:** Clean state, activate venv
 2. **Execute:** Run test commands
 3. **Verify:** Check results and assertions
 4. **Cleanup:** Stop server, remove resources
 ## Running Individual Tests
 ```bash
 # Make test executable (if needed)
 chmod +x deploy/docker/tests/cli/basic/test_01_start_default.sh
 # Run directly
 ./deploy/docker/tests/cli/basic/test_01_start_default.sh
 # Or use the test runner
 ./run_tests.sh basic 01
 ```
 ## Interpreting Results
 ### Success Output
 ```
 ✅ Test passed: [description]
 ```
 ### Failure Output
 ```
 ❌ Test failed: [error message]
 ```
 ### Warning Output
 ```
 ⚠️  Warning: [issue description]
 ```
 ## Common Issues
 ### Docker Not Running
 ```
 Error: Docker daemon not running
 Solution: Start Docker Desktop or Docker daemon
 ```
 ### Port Already In Use
 ```
 Error: Port 11235 is already in use
 Solution: Stop existing server or use different port
 ```
 ### Virtual Environment Not Found
 ```
 Warning: venv not found
 Solution: Create venv and activate it
 ```
 ### Playwright Not Installed
 ```
 Error: playwright module not found
 Solution: pip install playwright && playwright install chromium
 ```
 ## Test Development
 ### Adding New Tests
 1. **Choose category:** basic, advanced, resource, dashboard, or edge
 2. **Create test file:** Follow naming pattern `test_XX_description.sh`
 3. **Use template:**
 ```bash
 #!/bin/bash
 # Test: [Description]
 # Expected: [What should happen]
 set -e
 echo "=== Test: [Name] ==="
 echo ""
 source venv/bin/activate
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Test logic here
 # Cleanup
 crwl server stop >/dev/null 2>&1
 echo ""
 echo "✅ Test passed: [success message]"
 ```
 4. **Make executable:** `chmod +x test_XX_description.sh`
 5. **Test it:** `./test_XX_description.sh`
 6. **Add to runner:** Tests are auto-discovered by `run_tests.sh`
 ## CI/CD Integration
 These tests can be integrated into CI/CD pipelines:
 ```yaml
 # Example GitHub Actions
 - name: Run CLI Tests
  run: |
    source venv/bin/activate
    cd deploy/docker/tests/cli
    ./run_tests.sh all
 ```
 ## Performance Considerations
 - **Basic tests:** ~2-5 minutes total
 - **Advanced tests:** ~5-10 minutes total
 - **Resource tests:** ~10-15 minutes total (including 5-min stability test)
 - **Dashboard test:** ~3-5 minutes
 - **Edge case tests:** ~5-8 minutes total
 **Full suite:** ~30-45 minutes
 ## Best Practices
 1. **Always cleanup:** Each test should cleanup after itself
 2. **Wait for readiness:** Add sleep after starting servers
 3. **Check health:** Verify health endpoint before assertions
 4. **Graceful failures:** Use `|| true` to continue on expected failures
 5. **Clear messages:** Output should clearly indicate what's being tested
 ## Troubleshooting
 ### Tests Hanging
 - Check if Docker containers are stuck
 - Look for port conflicts
 - Verify network connectivity
 ### Intermittent Failures
 - Increase sleep durations for slower systems
 - Check system resources (memory, CPU)
 - Verify Docker has enough resources allocated
 ### All Tests Failing
 - Verify Docker is running: `docker ps`
 - Check CLI is installed: `which crwl`
 - Activate venv: `source venv/bin/activate`
 - Check server manager: `crwl server status`
 ## Contributing
 When adding new tests:
 1. Follow existing naming conventions
 2. Add comprehensive documentation
 3. Test on clean system
 4. Update this README
 5. Ensure cleanup is robust
 ## License
 Same as Crawl4AI project license.
--- a/deploy/docker/tests/cli/TEST_RESULTS.md
+++ b/deploy/docker/tests/cli/TEST_RESULTS.md
@@ -1,163 +0,0 @@
 # CLI Test Suite - Execution Results
 **Date:** 2025-10-20
 **Status:** ✅ PASSED
 ## Summary
 | Category | Total | Passed | Failed | Skipped |
 |----------|-------|--------|--------|---------|
 | Basic Tests | 8 | 8 | 0 | 0 |
 | Advanced Tests | 8 | 8 | 0 | 0 |
 | Edge Case Tests | 10 | 10 | 0 | 0 |
 | Resource Tests | 3 | 3 | 0 | 2 (skipped) |
 | Dashboard UI Tests | 0 | 0 | 0 | 1 (not run) |
 | **TOTAL** | **29** | **29** | **0** | **3** |
 **Success Rate:** 100% (29/29 tests passed)
 ## Test Results by Category
 ### ✅ Basic Tests (8/8 Passed)
 | Test | Status | Notes |
 |------|--------|-------|
 | test_01_start_default | ✅ PASS | Server starts with defaults (1 replica, port 11235) |
 | test_02_status | ✅ PASS | Status command shows correct information |
 | test_03_stop | ✅ PASS | Server stops cleanly, port freed |
 | test_04_start_custom_port | ✅ PASS | Server starts on port 8080 |
 | test_05_start_replicas | ✅ PASS | Compose mode with 3 replicas |
 | test_06_logs | ✅ PASS | Logs retrieved successfully |
 | test_07_restart | ✅ PASS | Server restarts preserving config (2 replicas) |
 | test_08_cleanup | ✅ PASS | Force cleanup removes all resources |
 ### ✅ Advanced Tests (8/8 Passed)
 | Test | Status | Notes |
 |------|--------|-------|
 | test_01_scale_up | ✅ PASS | Scaled 3 → 5 replicas successfully |
 | test_02_scale_down | ✅ PASS | Scaled 5 → 2 replicas successfully |
 | test_03_mode_single | ✅ PASS | Explicit single mode works |
 | test_04_mode_compose | ✅ PASS | Compose mode with 3 replicas and Nginx |
 | test_05_custom_image | ✅ PASS | Custom image specification works |
 | test_06_env_file | ✅ PASS | Environment file loading works |
 | test_07_stop_remove_volumes | ✅ PASS | Volumes handled during cleanup |
 | test_08_restart_with_scale | ✅ PASS | Restart with scale change (2 → 4 replicas) |
 ### ✅ Edge Case Tests (10/10 Passed)
 | Test | Status | Notes |
 |------|--------|-------|
 | test_01_already_running | ✅ PASS | Proper error for duplicate start |
 | test_02_not_running | ✅ PASS | Appropriate errors when server stopped |
 | test_03_scale_single_mode | ✅ PASS | Cannot scale single mode (expected error) |
 | test_04_invalid_port | ✅ PASS | Rejected ports: 0, -1, 99999, 65536 |
 | test_05_invalid_replicas | ✅ PASS | Rejected replicas: 0, -1, 101 |
 | test_06_missing_env_file | ✅ PASS | File not found error |
 | test_07_port_in_use | ✅ PASS | Port conflict detected |
 | test_08_state_corruption | ✅ PASS | Corrupted state handled gracefully |
 | test_09_network_conflict | ✅ PASS | Network collision handled |
 | test_10_rapid_operations | ✅ PASS | Rapid start/stop/restart cycles work |
 ### ✅ Resource Tests (3/5 Completed)
 | Test | Status | Notes |
 |------|--------|-------|
 | test_01_memory_monitoring | ✅ PASS | Baseline: 9.6%, After: 12.1%, Pool: 450 MB |
 | test_02_cpu_stress | ✅ PASS | Handled 10 concurrent requests |
 | test_03_max_replicas | ⏭️ SKIP | Takes ~2 minutes (10 replicas) |
 | test_04_cleanup_verification | ✅ PASS | All resources cleaned up |
 | test_05_long_running | ⏭️ SKIP | Takes 5 minutes |
 ### Dashboard UI Tests (Not Run)
 | Test | Status | Notes |
 |------|--------|-------|
 | test_01_dashboard_ui | ⏭️ SKIP | Requires Playwright, takes ~5 minutes |
 ## Key Findings
 ### ✅ Strengths
 1. **Robust Error Handling**
   - All invalid inputs properly rejected with clear error messages
   - State corruption detected and recovered automatically
   - Port conflicts identified before container start
 2. **Scaling Functionality**
   - Live scaling works smoothly (3 → 5 → 2 replicas)
   - Mode detection works correctly (single vs compose)
   - Restart preserves configuration
 3. **Resource Management**
   - Cleanup thoroughly removes all Docker resources
   - Memory usage reasonable (9.6% → 12.1% with 5 crawls)
   - Concurrent requests handled without errors
 4. **CLI Usability**
   - Clear, color-coded output
   - Helpful error messages with hints
   - Status command shows comprehensive info
 ### 📊 Performance Observations
 - **Startup Time:** ~5 seconds for single container, ~10-12 seconds for 3 replicas
 - **Memory Usage:** Baseline 9.6%, increases to 12.1% after 5 crawls
 - **Browser Pool:** ~450 MB memory usage (reasonable)
 - **Concurrent Load:** Successfully handled 10 parallel requests
 ### 🔧 Issues Found
 None! All 29 tests passed successfully.
 ## Test Execution Notes
 ### Test Environment
 - **OS:** macOS (Darwin 24.3.0)
 - **Docker:** Running
 - **Python:** Virtual environment activated
 - **Date:** 2025-10-20
 ### Skipped Tests Rationale
 1. **test_03_max_replicas:** Takes ~2 minutes to start 10 replicas
 2. **test_05_long_running:** 5-minute stability test
 3. **test_01_dashboard_ui:** Requires Playwright installation, UI screenshots
 These tests are fully implemented and can be run manually when time permits.
 ## Verification Commands
 All tests can be re-run with:
 ```bash
 # Individual test
 bash deploy/docker/tests/cli/basic/test_01_start_default.sh
 # Category
 ./deploy/docker/tests/cli/run_tests.sh basic
 # All tests
 ./deploy/docker/tests/cli/run_tests.sh all
 ```
 ## Conclusion
 ✅ **The CLI test suite is comprehensive and thoroughly validates all functionality.**
 - All core features tested and working
 - Error handling is robust
 - Edge cases properly covered
 - Resource management verified
 - No bugs or issues found
 The Crawl4AI Docker server CLI is production-ready with excellent test coverage.
 ---
 **Next Steps:**
 1. Run skipped tests when time permits (optional)
 2. Integrate into CI/CD pipeline
 3. Run dashboard UI test for visual verification
 4. Document test results in main README
 **Recommendation:** ✅ Ready for production use
--- a/deploy/docker/tests/cli/TEST_SUMMARY.md
+++ b/deploy/docker/tests/cli/TEST_SUMMARY.md
@@ -1,300 +0,0 @@
 # CLI Test Suite - Implementation Summary
 ## Completed Implementation
 Successfully created a comprehensive E2E test suite for the Crawl4AI Docker server CLI.
 ## Test Suite Overview
 ### Total Tests: 32
 #### 1. Basic Tests (8 tests) ✅
 - `test_01_start_default.sh` - Start with default settings
 - `test_02_status.sh` - Status command validation
 - `test_03_stop.sh` - Clean server shutdown
 - `test_04_start_custom_port.sh` - Custom port configuration
 - `test_05_start_replicas.sh` - Multi-replica deployment
 - `test_06_logs.sh` - Log retrieval
 - `test_07_restart.sh` - Server restart
 - `test_08_cleanup.sh` - Force cleanup
 #### 2. Advanced Tests (8 tests) ✅
 - `test_01_scale_up.sh` - Scale from 3 to 5 replicas
 - `test_02_scale_down.sh` - Scale from 5 to 2 replicas
 - `test_03_mode_single.sh` - Explicit single mode
 - `test_04_mode_compose.sh` - Compose mode with Nginx
 - `test_05_custom_image.sh` - Custom image specification
 - `test_06_env_file.sh` - Environment file loading
 - `test_07_stop_remove_volumes.sh` - Volume cleanup
 - `test_08_restart_with_scale.sh` - Restart with scale change
 #### 3. Resource Tests (5 tests) ✅
 - `test_01_memory_monitoring.sh` - Memory usage tracking
 - `test_02_cpu_stress.sh` - CPU stress with concurrent requests
 - `test_03_max_replicas.sh` - Maximum (10) replicas stress test
 - `test_04_cleanup_verification.sh` - Resource cleanup verification
 - `test_05_long_running.sh` - 5-minute stability test
 #### 4. Dashboard UI Test (1 test) ✅
 - `test_01_dashboard_ui.py` - Comprehensive Playwright test
  - Automated browser testing
  - Screenshot capture (7 screenshots per run)
  - UI element validation
  - Container filter testing
  - WebSocket connection verification
 #### 5. Edge Case Tests (10 tests) ✅
 - `test_01_already_running.sh` - Duplicate start attempt
 - `test_02_not_running.sh` - Operations on stopped server
 - `test_03_scale_single_mode.sh` - Invalid scaling operation
 - `test_04_invalid_port.sh` - Port validation (0, -1, 99999, 65536)
 - `test_05_invalid_replicas.sh` - Replica validation (0, -1, 101)
 - `test_06_missing_env_file.sh` - Non-existent env file
 - `test_07_port_in_use.sh` - Port conflict detection
 - `test_08_state_corruption.sh` - State file corruption recovery
 - `test_09_network_conflict.sh` - Docker network collision handling
 - `test_10_rapid_operations.sh` - Rapid start/stop cycles
 ## Test Infrastructure
 ### Master Test Runner (`run_tests.sh`)
 - Run all tests or specific categories
 - Color-coded output (green/red/yellow)
 - Test counters (passed/failed/skipped)
 - Summary statistics
 - Individual test execution support
 ### Documentation
 - `README.md` - Comprehensive test documentation
  - Test descriptions and expected results
  - Usage instructions
  - Troubleshooting guide
  - Best practices
  - CI/CD integration examples
 - `TEST_SUMMARY.md` - Implementation summary (this file)
 ## File Structure
 ```
 deploy/docker/tests/cli/
 ├── README.md                      # Main documentation
 ├── TEST_SUMMARY.md                # This summary
 ├── run_tests.sh                   # Master test runner
 │
 ├── basic/                         # Basic CLI tests
 │   ├── test_01_start_default.sh
 │   ├── test_02_status.sh
 │   ├── test_03_stop.sh
 │   ├── test_04_start_custom_port.sh
 │   ├── test_05_start_replicas.sh
 │   ├── test_06_logs.sh
 │   ├── test_07_restart.sh
 │   └── test_08_cleanup.sh
 │
 ├── advanced/                      # Advanced feature tests
 │   ├── test_01_scale_up.sh
 │   ├── test_02_scale_down.sh
 │   ├── test_03_mode_single.sh
 │   ├── test_04_mode_compose.sh
 │   ├── test_05_custom_image.sh
 │   ├── test_06_env_file.sh
 │   ├── test_07_stop_remove_volumes.sh
 │   └── test_08_restart_with_scale.sh
 │
 ├── resource/                      # Resource and stress tests
 │   ├── test_01_memory_monitoring.sh
 │   ├── test_02_cpu_stress.sh
 │   ├── test_03_max_replicas.sh
 │   ├── test_04_cleanup_verification.sh
 │   └── test_05_long_running.sh
 │
 ├── dashboard/                     # Dashboard UI tests
 │   ├── test_01_dashboard_ui.py
 │   ├── run_dashboard_test.sh
 │   └── screenshots/               # Auto-generated screenshots
 │
 └── edge/                          # Edge case tests
    ├── test_01_already_running.sh
    ├── test_02_not_running.sh
    ├── test_03_scale_single_mode.sh
    ├── test_04_invalid_port.sh
    ├── test_05_invalid_replicas.sh
    ├── test_06_missing_env_file.sh
    ├── test_07_port_in_use.sh
    ├── test_08_state_corruption.sh
    ├── test_09_network_conflict.sh
    └── test_10_rapid_operations.sh
 ```
 ## Usage Examples
 ### Run All Tests (except dashboard)
 ```bash
 ./run_tests.sh
 ```
 ### Run Specific Category
 ```bash
 ./run_tests.sh basic
 ./run_tests.sh advanced
 ./run_tests.sh resource
 ./run_tests.sh edge
 ```
 ### Run Dashboard Tests
 ```bash
 ./run_tests.sh dashboard
 # or
 ./dashboard/run_dashboard_test.sh
 ```
 ### Run Individual Test
 ```bash
 ./run_tests.sh basic 01
 ./run_tests.sh edge 05
 ```
 ### Direct Execution
 ```bash
 ./basic/test_01_start_default.sh
 ./edge/test_01_already_running.sh
 ```
 ## Test Verification
 The following tests have been verified working:
 - ✅ `test_01_start_default.sh` - PASSED
 - ✅ `test_02_status.sh` - PASSED
 - ✅ `test_03_stop.sh` - PASSED
 - ✅ `test_03_mode_single.sh` - PASSED
 - ✅ `test_01_already_running.sh` - PASSED
 - ✅ Master test runner - PASSED
 ## Key Features
 ### Robustness
 - Each test cleans up after itself
 - Handles expected failures gracefully
 - Waits for server readiness before assertions
 - Comprehensive error checking
 ### Clarity
 - Clear test descriptions
 - Colored output for easy interpretation
 - Detailed error messages
 - Progress indicators
 ### Completeness
 - Covers all CLI commands
 - Tests success and failure paths
 - Validates error messages
 - Checks resource cleanup
 ### Maintainability
 - Consistent structure across all tests
 - Well-documented code
 - Modular test design
 - Easy to add new tests
 ## Test Coverage
 ### CLI Commands Tested
 - ✅ `crwl server start` (all options)
 - ✅ `crwl server stop` (with/without volumes)
 - ✅ `crwl server status`
 - ✅ `crwl server scale`
 - ✅ `crwl server logs`
 - ✅ `crwl server restart`
 - ✅ `crwl server cleanup`
 ### Deployment Modes Tested
 - ✅ Single container mode
 - ✅ Compose mode (multi-container)
 - ✅ Auto mode detection
 ### Features Tested
 - ✅ Custom ports
 - ✅ Custom replicas (1-10)
 - ✅ Custom images
 - ✅ Environment files
 - ✅ Live scaling
 - ✅ Configuration persistence
 - ✅ Resource cleanup
 - ✅ Dashboard UI
 ### Error Handling Tested
 - ✅ Invalid inputs (ports, replicas)
 - ✅ Missing files
 - ✅ Port conflicts
 - ✅ State corruption
 - ✅ Network conflicts
 - ✅ Rapid operations
 - ✅ Duplicate operations
 ## Performance
 ### Estimated Execution Times
 - Basic tests: ~2-5 minutes
 - Advanced tests: ~5-10 minutes
 - Resource tests: ~10-15 minutes
 - Dashboard test: ~3-5 minutes
 - Edge case tests: ~5-8 minutes
 **Total: ~30-45 minutes for full suite**
 ## Next Steps
 ### Recommended Actions
 1. ✅ Run full test suite to verify all tests
 2. ✅ Test dashboard UI test with Playwright
 3. ✅ Verify long-running stability test
 4. ✅ Integrate into CI/CD pipeline
 5. ✅ Add to project documentation
 ### Future Enhancements
 - Add performance benchmarking
 - Add load testing scenarios
 - Add network failure simulation
 - Add disk space tests
 - Add security tests
 - Add multi-host tests (Swarm mode)
 ## Notes
 ### Dependencies
 - Docker running
 - Virtual environment activated
 - `jq` for JSON parsing (installed by default on most systems)
 - `bc` for calculations (installed by default on most systems)
 - Playwright for dashboard tests (optional)
 ### Test Philosophy
 - **Small:** Each test focuses on one specific aspect
 - **Smart:** Tests verify both success and failure paths
 - **Strong:** Robust cleanup and error handling
 - **Self-contained:** Each test is independent
 ### Known Limitations
 - Dashboard test requires Playwright installation
 - Long-running test takes 5 minutes
 - Max replicas test requires significant system resources
 - Some tests may need adjustment for slower systems
 ## Success Criteria
 ✅ All 32 tests created
 ✅ Test runner implemented
 ✅ Documentation complete
 ✅ Tests verified working
 ✅ File structure organized
 ✅ Error handling comprehensive
 ✅ Cleanup mechanisms robust
 ## Conclusion
 The CLI test suite is complete and ready for use. It provides comprehensive coverage of all CLI functionality, validates error handling, and ensures robustness across various scenarios.
 **Status:** ✅ COMPLETE
 **Date:** 2025-10-20
 **Tests:** 32 (8 basic + 8 advanced + 5 resource + 1 dashboard + 10 edge)
--- a/deploy/docker/tests/cli/advanced/test_01_scale_up.sh
+++ b/deploy/docker/tests/cli/advanced/test_01_scale_up.sh
@@ -1,56 +0,0 @@
 #!/bin/bash
 # Test: Scale server up from 3 to 5 replicas
 # Expected: Server scales without downtime
 set -e
 echo "=== Test: Scale Up (3 → 5 replicas) ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start with 3 replicas
 echo "Starting server with 3 replicas..."
 crwl server start --replicas 3 >/dev/null 2>&1
 sleep 10
 # Verify 3 replicas
 STATUS=$(crwl server status | grep "Replicas" || echo "")
 echo "Initial status: $STATUS"
 # Scale up to 5
 echo ""
 echo "Scaling up to 5 replicas..."
 crwl server scale 5
 sleep 10
 # Verify 5 replicas
 STATUS=$(crwl server status)
 echo "$STATUS"
 if ! echo "$STATUS" | grep -q "5"; then
    echo "❌ Status does not show 5 replicas"
    crwl server stop
    exit 1
 fi
 # Verify health during scaling
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "❌ Health check failed after scaling"
    crwl server stop
    exit 1
 fi
 # Cleanup
 echo "Cleaning up..."
 crwl server stop >/dev/null 2>&1
 echo ""
 echo "✅ Test passed: Successfully scaled from 3 to 5 replicas"
--- a/deploy/docker/tests/cli/advanced/test_02_scale_down.sh
+++ b/deploy/docker/tests/cli/advanced/test_02_scale_down.sh
@@ -1,56 +0,0 @@
 #!/bin/bash
 # Test: Scale server down from 5 to 2 replicas
 # Expected: Server scales down gracefully
 set -e
 echo "=== Test: Scale Down (5 → 2 replicas) ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start with 5 replicas
 echo "Starting server with 5 replicas..."
 crwl server start --replicas 5 >/dev/null 2>&1
 sleep 12
 # Verify 5 replicas
 STATUS=$(crwl server status | grep "Replicas" || echo "")
 echo "Initial status: $STATUS"
 # Scale down to 2
 echo ""
 echo "Scaling down to 2 replicas..."
 crwl server scale 2
 sleep 8
 # Verify 2 replicas
 STATUS=$(crwl server status)
 echo "$STATUS"
 if ! echo "$STATUS" | grep -q "2"; then
    echo "❌ Status does not show 2 replicas"
    crwl server stop
    exit 1
 fi
 # Verify health after scaling down
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "❌ Health check failed after scaling down"
    crwl server stop
    exit 1
 fi
 # Cleanup
 echo "Cleaning up..."
 crwl server stop >/dev/null 2>&1
 echo ""
 echo "✅ Test passed: Successfully scaled down from 5 to 2 replicas"
--- a/deploy/docker/tests/cli/advanced/test_03_mode_single.sh
+++ b/deploy/docker/tests/cli/advanced/test_03_mode_single.sh
@@ -1,52 +0,0 @@
 #!/bin/bash
 # Test: Start server explicitly in single mode
 # Expected: Server starts in single mode
 set -e
 echo "=== Test: Explicit Single Mode ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start in single mode explicitly
 echo "Starting server in single mode..."
 crwl server start --mode single
 sleep 5
 # Check mode
 STATUS=$(crwl server status)
 echo "$STATUS"
 if ! echo "$STATUS" | grep -q "single"; then
    echo "❌ Mode is not 'single'"
    crwl server stop
    exit 1
 fi
 if ! echo "$STATUS" | grep -q "1"; then
    echo "❌ Should have 1 replica in single mode"
    crwl server stop
    exit 1
 fi
 # Verify health
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "❌ Health check failed"
    crwl server stop
    exit 1
 fi
 # Cleanup
 echo "Cleaning up..."
 crwl server stop >/dev/null 2>&1
 echo ""
 echo "✅ Test passed: Server started in single mode"
--- a/deploy/docker/tests/cli/advanced/test_04_mode_compose.sh
+++ b/deploy/docker/tests/cli/advanced/test_04_mode_compose.sh
@@ -1,52 +0,0 @@
 #!/bin/bash
 # Test: Start server in compose mode with replicas
 # Expected: Server starts in compose mode with Nginx
 set -e
 echo "=== Test: Compose Mode with 3 Replicas ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start in compose mode
 echo "Starting server in compose mode with 3 replicas..."
 crwl server start --mode compose --replicas 3
 sleep 12
 # Check mode
 STATUS=$(crwl server status)
 echo "$STATUS"
 if ! echo "$STATUS" | grep -q "3"; then
    echo "❌ Status does not show 3 replicas"
    crwl server stop
    exit 1
 fi
 # Verify Nginx is running (load balancer)
 NGINX_RUNNING=$(docker ps --filter "name=nginx" --format "{{.Names}}" || echo "")
 if [[ -z "$NGINX_RUNNING" ]]; then
    echo "⚠️  Warning: Nginx load balancer not detected (may be using swarm or single mode)"
 fi
 # Verify health through load balancer
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "❌ Health check failed"
    crwl server stop
    exit 1
 fi
 # Cleanup
 echo "Cleaning up..."
 crwl server stop >/dev/null 2>&1
 echo ""
 echo "✅ Test passed: Server started in compose mode"
--- a/deploy/docker/tests/cli/advanced/test_05_custom_image.sh
+++ b/deploy/docker/tests/cli/advanced/test_05_custom_image.sh
@@ -1,47 +0,0 @@
 #!/bin/bash
 # Test: Start server with custom image tag
 # Expected: Server uses specified image
 set -e
 echo "=== Test: Custom Image Specification ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Use latest tag explicitly (or specify a different tag if available)
 IMAGE="unclecode/crawl4ai:latest"
 echo "Starting server with image: $IMAGE..."
 crwl server start --image "$IMAGE"
 sleep 5
 # Check status shows correct image
 STATUS=$(crwl server status)
 echo "$STATUS"
 if ! echo "$STATUS" | grep -q "crawl4ai"; then
    echo "❌ Status does not show correct image"
    crwl server stop
    exit 1
 fi
 # Verify health
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "❌ Health check failed"
    crwl server stop
    exit 1
 fi
 # Cleanup
 echo "Cleaning up..."
 crwl server stop >/dev/null 2>&1
 echo ""
 echo "✅ Test passed: Server started with custom image"
--- a/deploy/docker/tests/cli/advanced/test_06_env_file.sh
+++ b/deploy/docker/tests/cli/advanced/test_06_env_file.sh
@@ -1,47 +0,0 @@
 #!/bin/bash
 # Test: Start server with environment file
 # Expected: Server loads environment variables
 set -e
 echo "=== Test: Start with Environment File ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Create a test env file
 TEST_ENV_FILE="/tmp/test_crawl4ai.env"
 cat > "$TEST_ENV_FILE" <<EOF
 TEST_VAR=test_value
 OPENAI_API_KEY=sk-test-key
 EOF
 echo "Created test env file at $TEST_ENV_FILE"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start with env file
 echo "Starting server with env file..."
 crwl server start --env-file "$TEST_ENV_FILE"
 sleep 5
 # Verify server started
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "❌ Health check failed"
    rm -f "$TEST_ENV_FILE"
    crwl server stop
    exit 1
 fi
 # Cleanup
 echo "Cleaning up..."
 crwl server stop >/dev/null 2>&1
 rm -f "$TEST_ENV_FILE"
 echo ""
 echo "✅ Test passed: Server started with environment file"
--- a/deploy/docker/tests/cli/advanced/test_07_stop_remove_volumes.sh
+++ b/deploy/docker/tests/cli/advanced/test_07_stop_remove_volumes.sh
@@ -1,49 +0,0 @@
 #!/bin/bash
 # Test: Stop server with volume removal
 # Expected: Volumes are removed along with containers
 set -e
 echo "=== Test: Stop with Remove Volumes ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Start server (which may create volumes)
 echo "Starting server..."
 crwl server start --replicas 2 >/dev/null 2>&1
 sleep 8
 # Make some requests to populate data
 echo "Making requests to populate data..."
 curl -s -X POST http://localhost:11235/crawl \
  -H "Content-Type: application/json" \
  -d '{"urls": ["https://httpbin.org/html"], "crawler_config": {}}' > /dev/null || true
 sleep 2
 # Stop with volume removal (needs confirmation, so we'll use cleanup instead)
 echo "Stopping server with volume removal..."
 # Note: --remove-volumes requires confirmation, so we use cleanup --force
 crwl server cleanup --force >/dev/null 2>&1
 sleep 3
 # Verify volumes are removed
 echo "Checking for remaining volumes..."
 VOLUMES=$(docker volume ls --filter "name=crawl4ai" --format "{{.Name}}" || echo "")
 if [[ -n "$VOLUMES" ]]; then
    echo "⚠️  Warning: Some volumes still exist: $VOLUMES"
    echo "  (This may be expected if using system-wide volumes)"
 fi
 # Verify server is stopped
 STATUS=$(crwl server status | grep "No server" || echo "RUNNING")
 if [[ "$STATUS" == "RUNNING" ]]; then
    echo "❌ Server still running after stop"
    exit 1
 fi
 echo ""
 echo "✅ Test passed: Server stopped and volumes handled"
--- a/deploy/docker/tests/cli/advanced/test_08_restart_with_scale.sh
+++ b/deploy/docker/tests/cli/advanced/test_08_restart_with_scale.sh
@@ -1,56 +0,0 @@
 #!/bin/bash
 # Test: Restart server with different replica count
 # Expected: Server restarts with new replica count
 set -e
 echo "=== Test: Restart with Scale Change ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start with 2 replicas
 echo "Starting server with 2 replicas..."
 crwl server start --replicas 2 >/dev/null 2>&1
 sleep 8
 # Verify 2 replicas
 STATUS=$(crwl server status | grep "Replicas" || echo "")
 echo "Initial: $STATUS"
 # Restart with 4 replicas
 echo ""
 echo "Restarting with 4 replicas..."
 crwl server restart --replicas 4
 sleep 10
 # Verify 4 replicas
 STATUS=$(crwl server status)
 echo "$STATUS"
 if ! echo "$STATUS" | grep -q "4"; then
    echo "❌ Status does not show 4 replicas after restart"
    crwl server stop
    exit 1
 fi
 # Verify health
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "❌ Health check failed after restart"
    crwl server stop
    exit 1
 fi
 # Cleanup
 echo "Cleaning up..."
 crwl server stop >/dev/null 2>&1
 echo ""
 echo "✅ Test passed: Server restarted with new replica count"
--- a/deploy/docker/tests/cli/basic/test_01_start_default.sh
+++ b/deploy/docker/tests/cli/basic/test_01_start_default.sh
@@ -1,52 +0,0 @@
 #!/bin/bash
 # Test: Start server with default settings
 # Expected: Server starts with 1 replica on port 11235
 set -e
 echo "=== Test: Start Server with Defaults ==="
 echo "Expected: 1 replica, port 11235, auto mode"
 echo ""
 # Activate virtual environment
 # Navigate to project root and activate venv
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup any existing server
 echo "Cleaning up any existing server..."
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start server with defaults
 echo "Starting server with default settings..."
 crwl server start
 # Wait for server to be ready
 echo "Waiting for server to be healthy..."
 sleep 5
 # Verify server is running
 echo "Checking server status..."
 STATUS=$(crwl server status | grep "Running" || echo "NOT_RUNNING")
 if [[ "$STATUS" == "NOT_RUNNING" ]]; then
    echo "❌ Server failed to start"
    crwl server stop
    exit 1
 fi
 # Check health endpoint
 echo "Checking health endpoint..."
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "❌ Health check failed: $HEALTH"
    crwl server stop
    exit 1
 fi
 # Cleanup
 echo "Cleaning up..."
 crwl server stop
 echo ""
 echo "✅ Test passed: Server started with defaults and responded to health check"
--- a/deploy/docker/tests/cli/basic/test_02_status.sh
+++ b/deploy/docker/tests/cli/basic/test_02_status.sh
@@ -1,42 +0,0 @@
 #!/bin/bash
 # Test: Check server status command
 # Expected: Shows running status with correct details
 set -e
 echo "=== Test: Server Status Command ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Start server first
 echo "Starting server..."
 crwl server start >/dev/null 2>&1
 sleep 5
 # Check status
 echo "Checking server status..."
 STATUS_OUTPUT=$(crwl server status)
 echo "$STATUS_OUTPUT"
 echo ""
 # Verify output contains expected fields
 if ! echo "$STATUS_OUTPUT" | grep -q "Running"; then
    echo "❌ Status does not show 'Running'"
    crwl server stop
    exit 1
 fi
 if ! echo "$STATUS_OUTPUT" | grep -q "11235"; then
    echo "❌ Status does not show correct port"
    crwl server stop
    exit 1
 fi
 # Cleanup
 echo "Cleaning up..."
 crwl server stop >/dev/null 2>&1
 echo ""
 echo "✅ Test passed: Status command shows correct information"
--- a/deploy/docker/tests/cli/basic/test_03_stop.sh
+++ b/deploy/docker/tests/cli/basic/test_03_stop.sh
@@ -1,45 +0,0 @@
 #!/bin/bash
 # Test: Stop server command
 # Expected: Server stops cleanly and port becomes available
 set -e
 echo "=== Test: Stop Server Command ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Start server first
 echo "Starting server..."
 crwl server start >/dev/null 2>&1
 sleep 5
 # Verify running
 echo "Verifying server is running..."
 if ! curl -s http://localhost:11235/health > /dev/null 2>&1; then
    echo "❌ Server is not running before stop"
    exit 1
 fi
 # Stop server
 echo "Stopping server..."
 crwl server stop
 # Verify stopped
 echo "Verifying server is stopped..."
 sleep 3
 if curl -s http://localhost:11235/health > /dev/null 2>&1; then
    echo "❌ Server is still responding after stop"
    exit 1
 fi
 # Check status shows not running
 STATUS=$(crwl server status | grep "No server" || echo "RUNNING")
 if [[ "$STATUS" == "RUNNING" ]]; then
    echo "❌ Status still shows server as running"
    exit 1
 fi
 echo ""
 echo "✅ Test passed: Server stopped cleanly"
--- a/deploy/docker/tests/cli/basic/test_04_start_custom_port.sh
+++ b/deploy/docker/tests/cli/basic/test_04_start_custom_port.sh
@@ -1,46 +0,0 @@
 #!/bin/bash
 # Test: Start server with custom port
 # Expected: Server starts on port 8080 instead of default 11235
 set -e
 echo "=== Test: Start Server with Custom Port ==="
 echo "Expected: Server on port 8080"
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start on custom port
 echo "Starting server on port 8080..."
 crwl server start --port 8080
 sleep 5
 # Check health on custom port
 echo "Checking health on port 8080..."
 HEALTH=$(curl -s http://localhost:8080/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "❌ Health check failed on port 8080: $HEALTH"
    crwl server stop
    exit 1
 fi
 # Verify default port is NOT responding
 echo "Verifying port 11235 is not in use..."
 if curl -s http://localhost:11235/health > /dev/null 2>&1; then
    echo "❌ Server is also running on default port 11235"
    crwl server stop
    exit 1
 fi
 # Cleanup
 echo "Cleaning up..."
 crwl server stop
 echo ""
 echo "✅ Test passed: Server started on custom port 8080"
--- a/deploy/docker/tests/cli/basic/test_05_start_replicas.sh
+++ b/deploy/docker/tests/cli/basic/test_05_start_replicas.sh
@@ -1,54 +0,0 @@
 #!/bin/bash
 # Test: Start server with multiple replicas
 # Expected: Server starts with 3 replicas in compose mode
 set -e
 echo "=== Test: Start Server with 3 Replicas ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start with 3 replicas
 echo "Starting server with 3 replicas..."
 crwl server start --replicas 3
 sleep 10
 # Check status shows 3 replicas
 echo "Checking status..."
 STATUS_OUTPUT=$(crwl server status)
 echo "$STATUS_OUTPUT"
 if ! echo "$STATUS_OUTPUT" | grep -q "3"; then
    echo "❌ Status does not show 3 replicas"
    crwl server stop
    exit 1
 fi
 # Check health endpoint
 echo "Checking health endpoint..."
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "❌ Health check failed"
    crwl server stop
    exit 1
 fi
 # Check container discovery (should show 3 containers eventually)
 echo "Checking container discovery..."
 sleep 5  # Wait for heartbeats
 CONTAINERS=$(curl -s http://localhost:11235/monitor/containers | jq -r '.count' 2>/dev/null || echo "0")
 echo "Container count: $CONTAINERS"
 # Cleanup
 echo "Cleaning up..."
 crwl server stop
 echo ""
 echo "✅ Test passed: Server started with 3 replicas"
--- a/deploy/docker/tests/cli/basic/test_06_logs.sh
+++ b/deploy/docker/tests/cli/basic/test_06_logs.sh
@@ -1,47 +0,0 @@
 #!/bin/bash
 # Test: View server logs
 # Expected: Logs are displayed without errors
 set -e
 echo "=== Test: Server Logs Command ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Start server
 echo "Starting server..."
 crwl server start >/dev/null 2>&1
 sleep 5
 # Make a request to generate some logs
 echo "Making request to generate logs..."
 curl -s http://localhost:11235/health > /dev/null
 # Check logs (tail)
 echo "Fetching logs (last 50 lines)..."
 LOGS=$(crwl server logs --tail 50 2>&1 || echo "ERROR")
 if [[ "$LOGS" == "ERROR" ]]; then
    echo "❌ Failed to retrieve logs"
    crwl server stop
    exit 1
 fi
 echo "Log sample (first 10 lines):"
 echo "$LOGS" | head -n 10
 echo ""
 # Verify logs contain something (not empty)
 if [[ -z "$LOGS" ]]; then
    echo "❌ Logs are empty"
    crwl server stop
    exit 1
 fi
 # Cleanup
 echo "Cleaning up..."
 crwl server stop >/dev/null 2>&1
 echo ""
 echo "✅ Test passed: Logs retrieved successfully"
--- a/deploy/docker/tests/cli/basic/test_07_restart.sh
+++ b/deploy/docker/tests/cli/basic/test_07_restart.sh
@@ -1,55 +0,0 @@
 #!/bin/bash
 # Test: Restart server command
 # Expected: Server restarts with same configuration
 set -e
 echo "=== Test: Restart Server Command ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Start server with specific config
 echo "Starting server with 2 replicas..."
 crwl server start --replicas 2 >/dev/null 2>&1
 sleep 8
 # Get initial container ID
 echo "Getting initial state..."
 INITIAL_STATUS=$(crwl server status)
 echo "$INITIAL_STATUS"
 # Restart
 echo ""
 echo "Restarting server..."
 crwl server restart
 sleep 8
 # Check status after restart
 echo "Checking status after restart..."
 RESTART_STATUS=$(crwl server status)
 echo "$RESTART_STATUS"
 # Verify still has 2 replicas
 if ! echo "$RESTART_STATUS" | grep -q "2"; then
    echo "❌ Replica count not preserved after restart"
    crwl server stop
    exit 1
 fi
 # Verify health
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "❌ Health check failed after restart"
    crwl server stop
    exit 1
 fi
 # Cleanup
 echo "Cleaning up..."
 crwl server stop >/dev/null 2>&1
 echo ""
 echo "✅ Test passed: Server restarted with preserved configuration"
--- a/deploy/docker/tests/cli/basic/test_08_cleanup.sh
+++ b/deploy/docker/tests/cli/basic/test_08_cleanup.sh
@@ -1,46 +0,0 @@
 #!/bin/bash
 # Test: Force cleanup command
 # Expected: All resources removed even if state is corrupted
 set -e
 echo "=== Test: Force Cleanup Command ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Start server
 echo "Starting server..."
 crwl server start >/dev/null 2>&1
 sleep 5
 # Run cleanup (will prompt, so use force flag)
 echo "Running force cleanup..."
 crwl server cleanup --force
 sleep 3
 # Verify no containers running
 echo "Verifying cleanup..."
 CONTAINERS=$(docker ps --filter "name=crawl4ai" --format "{{.Names}}" || echo "")
 if [[ -n "$CONTAINERS" ]]; then
    echo "❌ Crawl4AI containers still running: $CONTAINERS"
    exit 1
 fi
 # Verify port is free
 if curl -s http://localhost:11235/health > /dev/null 2>&1; then
    echo "❌ Server still responding after cleanup"
    exit 1
 fi
 # Verify status shows not running
 STATUS=$(crwl server status | grep "No server" || echo "RUNNING")
 if [[ "$STATUS" == "RUNNING" ]]; then
    echo "❌ Status still shows server running after cleanup"
    exit 1
 fi
 echo ""
 echo "✅ Test passed: Force cleanup removed all resources"
--- a/deploy/docker/tests/cli/dashboard/run_dashboard_test.sh
+++ b/deploy/docker/tests/cli/dashboard/run_dashboard_test.sh
@@ -1,27 +0,0 @@
 #!/bin/bash
 # Wrapper script to run dashboard UI test with proper environment
 set -e
 echo "=== Dashboard UI Test ==="
 echo ""
 # Activate virtual environment
 source venv/bin/activate
 # Make sure playwright is installed
 echo "Checking Playwright installation..."
 python -c "import playwright" 2>/dev/null || {
    echo "Installing Playwright..."
    pip install playwright
    playwright install chromium
 }
 # Run the test
 echo ""
 echo "Running dashboard UI test..."
 python deploy/docker/tests/cli/dashboard/test_01_dashboard_ui.py
 echo ""
 echo "✅ Dashboard test complete"
 echo "Check deploy/docker/tests/cli/dashboard/screenshots/ for results"
--- a/deploy/docker/tests/cli/dashboard/test_01_dashboard_ui.py
+++ b/deploy/docker/tests/cli/dashboard/test_01_dashboard_ui.py
@@ -1,225 +0,0 @@
 #!/usr/bin/env python3
 """
 Dashboard UI Test with Playwright
 Tests the monitoring dashboard UI functionality
 """
 import asyncio
 import subprocess
 import time
 import os
 from pathlib import Path
 from playwright.async_api import async_playwright
 BASE_URL = "http://localhost:11235"
 SCREENSHOT_DIR = Path(__file__).parent / "screenshots"
 async def start_server():
    """Start server with 3 replicas"""
    print("Starting server with 3 replicas...")
    subprocess.run(["crwl", "server", "stop"],
                   stdout=subprocess.DEVNULL,
                   stderr=subprocess.DEVNULL)
    time.sleep(2)
    result = subprocess.run(
        ["crwl", "server", "start", "--replicas", "3"],
        capture_output=True,
        text=True
    )
    if result.returncode != 0:
        raise Exception(f"Failed to start server: {result.stderr}")
    print("Waiting for server to be ready...")
    time.sleep(12)
 async def run_demo_script():
    """Run the demo script in background to generate activity"""
    print("Starting demo script to generate dashboard activity...")
    demo_path = Path(__file__).parent.parent.parent / "monitor" / "demo_monitor_dashboard.py"
    process = subprocess.Popen(
        ["python", str(demo_path)],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
    # Let it run for a bit to generate some data
    print("Waiting for demo to generate data...")
    time.sleep(10)
    return process
 async def test_dashboard_ui():
    """Test dashboard UI with Playwright"""
    # Create screenshot directory
    SCREENSHOT_DIR.mkdir(exist_ok=True)
    print(f"Screenshots will be saved to: {SCREENSHOT_DIR}")
    async with async_playwright() as p:
        # Launch browser
        print("\nLaunching browser...")
        browser = await p.chromium.launch(headless=True)
        context = await browser.new_context(viewport={'width': 1920, 'height': 1080})
        page = await context.new_page()
        try:
            # Navigate to dashboard
            print(f"Navigating to {BASE_URL}/dashboard")
            await page.goto(f"{BASE_URL}/dashboard", wait_until="networkidle")
            await asyncio.sleep(3)
            # Take full dashboard screenshot
            print("Taking full dashboard screenshot...")
            await page.screenshot(path=SCREENSHOT_DIR / "01_full_dashboard.png", full_page=True)
            print(f"  ✅ Saved: 01_full_dashboard.png")
            # Verify page title
            title = await page.title()
            print(f"\nPage title: {title}")
            if "Monitor" not in title and "Dashboard" not in title:
                print("  ⚠️  Warning: Title doesn't contain 'Monitor' or 'Dashboard'")
            # Check for infrastructure card (container filters)
            print("\nChecking Infrastructure card...")
            infrastructure = await page.query_selector('.card h3:has-text("Infrastructure")')
            if infrastructure:
                print("  ✅ Infrastructure card found")
                await page.screenshot(path=SCREENSHOT_DIR / "02_infrastructure_card.png")
                print(f"  ✅ Saved: 02_infrastructure_card.png")
            else:
                print("  ❌ Infrastructure card not found")
            # Check for container filter buttons (All, C-1, C-2, C-3)
            print("\nChecking container filter buttons...")
            all_button = await page.query_selector('.filter-btn[data-container="all"]')
            if all_button:
                print("  ✅ 'All' filter button found")
                # Take screenshot of filter area
                await all_button.screenshot(path=SCREENSHOT_DIR / "03_filter_buttons.png")
                print(f"  ✅ Saved: 03_filter_buttons.png")
                # Test clicking filter button
                await all_button.click()
                await asyncio.sleep(1)
                print("  ✅ Clicked 'All' filter button")
            else:
                print("  ⚠️  'All' filter button not found (may appear after containers register)")
            # Check for WebSocket connection indicator
            print("\nChecking WebSocket connection...")
            ws_indicator = await page.query_selector('.ws-status, .connection-status, [class*="websocket"]')
            if ws_indicator:
                print("  ✅ WebSocket indicator found")
            else:
                print("  ⚠️  WebSocket indicator not found in DOM")
            # Check for main dashboard sections
            print("\nChecking dashboard sections...")
            sections = [
                ("Active Requests", ".active-requests, [class*='active']"),
                ("Completed Requests", ".completed-requests, [class*='completed']"),
                ("Browsers", ".browsers, [class*='browser']"),
                ("Timeline", ".timeline, [class*='timeline']"),
            ]
            for section_name, selector in sections:
                element = await page.query_selector(selector)
                if element:
                    print(f"  ✅ {section_name} section found")
                else:
                    print(f"  ⚠️  {section_name} section not found with selector: {selector}")
            # Scroll to different sections and take screenshots
            print("\nTaking section screenshots...")
            # Requests section
            requests = await page.query_selector('.card h3:has-text("Requests")')
            if requests:
                await requests.scroll_into_view_if_needed()
                await asyncio.sleep(1)
                await page.screenshot(path=SCREENSHOT_DIR / "04_requests_section.png")
                print(f"  ✅ Saved: 04_requests_section.png")
            # Browsers section
            browsers = await page.query_selector('.card h3:has-text("Browsers")')
            if browsers:
                await browsers.scroll_into_view_if_needed()
                await asyncio.sleep(1)
                await page.screenshot(path=SCREENSHOT_DIR / "05_browsers_section.png")
                print(f"  ✅ Saved: 05_browsers_section.png")
            # Timeline section
            timeline = await page.query_selector('.card h3:has-text("Timeline")')
            if timeline:
                await timeline.scroll_into_view_if_needed()
                await asyncio.sleep(1)
                await page.screenshot(path=SCREENSHOT_DIR / "06_timeline_section.png")
                print(f"  ✅ Saved: 06_timeline_section.png")
            # Check for tabs (if they exist)
            print("\nChecking for tabs...")
            tabs = await page.query_selector_all('.tab, [role="tab"]')
            if tabs:
                print(f"  ✅ Found {len(tabs)} tabs")
                for i, tab in enumerate(tabs[:5]):  # Check first 5 tabs
                    tab_text = await tab.inner_text()
                    print(f"    - Tab {i+1}: {tab_text}")
            else:
                print("  ℹ️  No tab elements found")
            # Wait for any animations to complete
            await asyncio.sleep(2)
            # Take final screenshot
            print("\nTaking final screenshot...")
            await page.screenshot(path=SCREENSHOT_DIR / "07_final_state.png", full_page=True)
            print(f"  ✅ Saved: 07_final_state.png")
            print("\n" + "="*60)
            print("Dashboard UI Test Complete!")
            print(f"Screenshots saved to: {SCREENSHOT_DIR}")
            print("="*60)
        finally:
            await browser.close()
 async def cleanup():
    """Stop server and cleanup"""
    print("\nCleaning up...")
    subprocess.run(["crwl", "server", "stop"],
                   stdout=subprocess.DEVNULL,
                   stderr=subprocess.DEVNULL)
    print("✅ Cleanup complete")
 async def main():
    """Main test execution"""
    demo_process = None
    try:
        # Start server
        await start_server()
        # Run demo script to generate activity
        demo_process = await run_demo_script()
        # Run dashboard UI test
        await test_dashboard_ui()
        print("\n✅ All dashboard UI tests passed!")
    except Exception as e:
        print(f"\n❌ Test failed: {e}")
        raise
    finally:
        # Stop demo script
        if demo_process:
            demo_process.terminate()
            demo_process.wait(timeout=5)
        # Cleanup server
        await cleanup()
 if __name__ == "__main__":
    asyncio.run(main())
--- a/deploy/docker/tests/cli/edge/test_01_already_running.sh
+++ b/deploy/docker/tests/cli/edge/test_01_already_running.sh
@@ -1,48 +0,0 @@
 #!/bin/bash
 # Test: Try starting server when already running
 # Expected: Error message indicating server is already running
 set -e
 echo "=== Test: Start When Already Running ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start server
 echo "Starting server..."
 crwl server start >/dev/null 2>&1
 sleep 5
 # Try to start again
 echo ""
 echo "Attempting to start server again (should fail)..."
 OUTPUT=$(crwl server start 2>&1 || true)
 echo "$OUTPUT"
 # Verify error message
 if echo "$OUTPUT" | grep -iq "already running"; then
    echo ""
    echo "✅ Test passed: Proper error for already running server"
 else
    echo ""
    echo "❌ Test failed: Expected 'already running' error message"
    crwl server stop
    exit 1
 fi
 # Verify original server still running
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "❌ Original server is not running"
    crwl server stop
    exit 1
 fi
 # Cleanup
 crwl server stop >/dev/null 2>&1
--- a/deploy/docker/tests/cli/edge/test_02_not_running.sh
+++ b/deploy/docker/tests/cli/edge/test_02_not_running.sh
@@ -1,50 +0,0 @@
 #!/bin/bash
 # Test: Operations when server is not running
 # Expected: Appropriate error messages
 set -e
 echo "=== Test: Operations When Not Running ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Make sure nothing is running
 crwl server stop 2>/dev/null || true
 sleep 2
 # Try status when not running
 echo "Checking status when not running..."
 OUTPUT=$(crwl server status 2>&1 || true)
 echo "$OUTPUT"
 echo ""
 if ! echo "$OUTPUT" | grep -iq "no server"; then
    echo "❌ Status should indicate no server running"
    exit 1
 fi
 # Try stop when not running
 echo "Trying to stop when not running..."
 OUTPUT=$(crwl server stop 2>&1 || true)
 echo "$OUTPUT"
 echo ""
 if ! echo "$OUTPUT" | grep -iq "no server\|not running"; then
    echo "❌ Stop should indicate no server running"
    exit 1
 fi
 # Try scale when not running
 echo "Trying to scale when not running..."
 OUTPUT=$(crwl server scale 3 2>&1 || true)
 echo "$OUTPUT"
 echo ""
 if ! echo "$OUTPUT" | grep -iq "no server\|not running"; then
    echo "❌ Scale should indicate no server running"
    exit 1
 fi
 echo "✅ Test passed: Appropriate errors for operations when not running"
--- a/deploy/docker/tests/cli/edge/test_03_scale_single_mode.sh
+++ b/deploy/docker/tests/cli/edge/test_03_scale_single_mode.sh
@@ -1,47 +0,0 @@
 #!/bin/bash
 # Test: Try to scale single container mode
 # Expected: Error indicating single mode cannot be scaled
 set -e
 echo "=== Test: Scale Single Container Mode ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start in single mode
 echo "Starting in single mode..."
 crwl server start --mode single >/dev/null 2>&1
 sleep 5
 # Try to scale
 echo ""
 echo "Attempting to scale single mode (should fail)..."
 OUTPUT=$(crwl server scale 3 2>&1 || true)
 echo "$OUTPUT"
 echo ""
 # Verify error message
 if echo "$OUTPUT" | grep -iq "single"; then
    echo "✅ Test passed: Proper error for scaling single mode"
 else
    echo "❌ Test failed: Expected error about single mode"
    crwl server stop
    exit 1
 fi
 # Verify server still running
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "❌ Server is not running after failed scale"
    crwl server stop
    exit 1
 fi
 # Cleanup
 crwl server stop >/dev/null 2>&1
--- a/deploy/docker/tests/cli/edge/test_04_invalid_port.sh
+++ b/deploy/docker/tests/cli/edge/test_04_invalid_port.sh
@@ -1,36 +0,0 @@
 #!/bin/bash
 # Test: Invalid port numbers
 # Expected: Validation errors for invalid ports
 set -e
 echo "=== Test: Invalid Port Numbers ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Test invalid ports
 INVALID_PORTS=(0 -1 99999 65536)
 for PORT in "${INVALID_PORTS[@]}"; do
    echo "Testing invalid port: $PORT"
    OUTPUT=$(crwl server start --port $PORT 2>&1 || true)
    if echo "$OUTPUT" | grep -iq "error\|invalid\|usage"; then
        echo "  ✅ Rejected port $PORT"
    else
        echo "  ⚠️  Port $PORT may have been accepted (output: $OUTPUT)"
    fi
    # Make sure no server started
    crwl server stop 2>/dev/null || true
    sleep 1
    echo ""
 done
 echo "✅ Test passed: Invalid ports handled appropriately"
--- a/deploy/docker/tests/cli/edge/test_05_invalid_replicas.sh
+++ b/deploy/docker/tests/cli/edge/test_05_invalid_replicas.sh
@@ -1,57 +0,0 @@
 #!/bin/bash
 # Test: Invalid replica counts
 # Expected: Validation errors for invalid replicas
 set -e
 echo "=== Test: Invalid Replica Counts ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Test invalid replica counts
 INVALID_REPLICAS=(0 -1 101)
 for REPLICAS in "${INVALID_REPLICAS[@]}"; do
    echo "Testing invalid replica count: $REPLICAS"
    OUTPUT=$(crwl server start --replicas $REPLICAS 2>&1 || true)
    if echo "$OUTPUT" | grep -iq "error\|invalid\|usage"; then
        echo "  ✅ Rejected replica count $REPLICAS"
    else
        echo "  ⚠️  Replica count $REPLICAS may have been accepted"
    fi
    # Make sure no server started
    crwl server stop 2>/dev/null || true
    sleep 1
    echo ""
 done
 # Test scaling to invalid counts
 echo "Testing scale to invalid counts..."
 crwl server start --replicas 2 >/dev/null 2>&1
 sleep 5
 INVALID_SCALE=(0 -1)
 for SCALE in "${INVALID_SCALE[@]}"; do
    echo "Testing scale to: $SCALE"
    OUTPUT=$(crwl server scale $SCALE 2>&1 || true)
    if echo "$OUTPUT" | grep -iq "error\|invalid\|must be at least 1"; then
        echo "  ✅ Rejected scale to $SCALE"
    else
        echo "  ⚠️  Scale to $SCALE may have been accepted"
    fi
    echo ""
 done
 # Cleanup
 crwl server stop >/dev/null 2>&1
 echo "✅ Test passed: Invalid replica counts handled appropriately"
--- a/deploy/docker/tests/cli/edge/test_06_missing_env_file.sh
+++ b/deploy/docker/tests/cli/edge/test_06_missing_env_file.sh
@@ -1,40 +0,0 @@
 #!/bin/bash
 # Test: Non-existent environment file
 # Expected: Error indicating file not found
 set -e
 echo "=== Test: Missing Environment File ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Try with non-existent file
 FAKE_FILE="/tmp/nonexistent_$(date +%s).env"
 echo "Attempting to start with non-existent env file: $FAKE_FILE"
 OUTPUT=$(crwl server start --env-file "$FAKE_FILE" 2>&1 || true)
 echo "$OUTPUT"
 echo ""
 # Verify error
 if echo "$OUTPUT" | grep -iq "error\|does not exist\|not found\|no such file"; then
    echo "✅ Test passed: Proper error for missing env file"
 else
    echo "❌ Test failed: Expected error about missing file"
    crwl server stop
    exit 1
 fi
 # Make sure no server started
 if curl -s http://localhost:11235/health > /dev/null 2>&1; then
    echo "❌ Server should not have started"
    crwl server stop
    exit 1
 fi
 echo "✅ Server correctly refused to start with missing env file"
--- a/deploy/docker/tests/cli/edge/test_07_port_in_use.sh
+++ b/deploy/docker/tests/cli/edge/test_07_port_in_use.sh
@@ -1,50 +0,0 @@
 #!/bin/bash
 # Test: Port already in use
 # Expected: Error indicating port is occupied
 set -e
 echo "=== Test: Port Already In Use ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start a simple HTTP server on port 11235 to occupy it
 echo "Starting dummy server on port 11235..."
 python -m http.server 11235 >/dev/null 2>&1 &
 DUMMY_PID=$!
 sleep 2
 # Try to start crawl4ai on same port
 echo "Attempting to start Crawl4AI on occupied port..."
 OUTPUT=$(crwl server start 2>&1 || true)
 echo "$OUTPUT"
 echo ""
 # Kill dummy server
 kill $DUMMY_PID 2>/dev/null || true
 sleep 1
 # Verify error message
 if echo "$OUTPUT" | grep -iq "port.*in use\|already in use\|address already in use"; then
    echo "✅ Test passed: Proper error for port in use"
 else
    echo "⚠️  Expected 'port in use' error (output may vary)"
 fi
 # Make sure Crawl4AI didn't start
 if curl -s http://localhost:11235/health > /dev/null 2>&1; then
    HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "unknown")
    if [[ "$HEALTH" == "ok" ]]; then
        echo "❌ Crawl4AI started despite port being occupied"
        crwl server stop
        exit 1
    fi
 fi
 echo "✅ Crawl4AI correctly refused to start on occupied port"
--- a/deploy/docker/tests/cli/edge/test_08_state_corruption.sh
+++ b/deploy/docker/tests/cli/edge/test_08_state_corruption.sh
@@ -1,79 +0,0 @@
 #!/bin/bash
 # Test: Corrupted state file
 # Expected: Cleanup recovers from corrupted state
 set -e
 echo "=== Test: State File Corruption ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start server to create state
 echo "Starting server to create state..."
 crwl server start >/dev/null 2>&1
 sleep 5
 # Get state file path
 STATE_FILE="$HOME/.crawl4ai/server/state.json"
 echo "State file: $STATE_FILE"
 # Verify state file exists
 if [[ ! -f "$STATE_FILE" ]]; then
    echo "❌ State file not created"
    crwl server stop
    exit 1
 fi
 echo "Original state:"
 cat "$STATE_FILE" | jq '.' || cat "$STATE_FILE"
 echo ""
 # Stop server
 crwl server stop >/dev/null 2>&1
 sleep 2
 # Corrupt state file
 echo "Corrupting state file..."
 echo "{ invalid json }" > "$STATE_FILE"
 cat "$STATE_FILE"
 echo ""
 # Try to start server (should handle corrupted state)
 echo "Attempting to start with corrupted state..."
 OUTPUT=$(crwl server start 2>&1 || true)
 echo "$OUTPUT"
 echo ""
 # Check if server started or gave clear error
 if curl -s http://localhost:11235/health > /dev/null 2>&1; then
    echo "✅ Server started despite corrupted state"
    crwl server stop
 elif echo "$OUTPUT" | grep -iq "already running"; then
    # State thinks server is running, use cleanup
    echo "State thinks server is running, using cleanup..."
    crwl server cleanup --force >/dev/null 2>&1
    sleep 2
    # Try starting again
    crwl server start >/dev/null 2>&1
    sleep 5
    if curl -s http://localhost:11235/health > /dev/null 2>&1; then
        echo "✅ Cleanup recovered from corrupted state"
        crwl server stop
    else
        echo "❌ Failed to recover from corrupted state"
        exit 1
    fi
 else
    echo "✅ Handled corrupted state appropriately"
 fi
 echo ""
 echo "✅ Test passed: System handles state corruption"
--- a/deploy/docker/tests/cli/edge/test_09_network_conflict.sh
+++ b/deploy/docker/tests/cli/edge/test_09_network_conflict.sh
@@ -1,47 +0,0 @@
 #!/bin/bash
 # Test: Docker network name collision
 # Expected: Handles existing network gracefully
 set -e
 echo "=== Test: Network Name Conflict ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Create a network with similar name
 NETWORK_NAME="crawl4ai_test_net"
 echo "Creating test network: $NETWORK_NAME..."
 docker network create "$NETWORK_NAME" 2>/dev/null || echo "Network may already exist"
 # Start server (should either use existing network or create its own)
 echo ""
 echo "Starting server..."
 crwl server start >/dev/null 2>&1
 sleep 5
 # Verify server started successfully
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "❌ Server failed to start"
    docker network rm "$NETWORK_NAME" 2>/dev/null || true
    crwl server stop
    exit 1
 fi
 echo "✅ Server started successfully despite network conflict"
 # Cleanup
 crwl server stop >/dev/null 2>&1
 sleep 2
 # Remove test network
 docker network rm "$NETWORK_NAME" 2>/dev/null || echo "Network already removed"
 echo ""
 echo "✅ Test passed: Handled network conflict gracefully"
--- a/deploy/docker/tests/cli/edge/test_10_rapid_operations.sh
+++ b/deploy/docker/tests/cli/edge/test_10_rapid_operations.sh
@@ -1,72 +0,0 @@
 #!/bin/bash
 # Test: Rapid start/stop/restart operations
 # Expected: System handles rapid operations without corruption
 set -e
 echo "=== Test: Rapid Operations ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Test 1: Rapid start/stop
 echo "Test 1: Rapid start/stop cycles..."
 for i in {1..3}; do
    echo "  Cycle $i/3..."
    crwl server start >/dev/null 2>&1
    sleep 3
    crwl server stop >/dev/null 2>&1
    sleep 2
 done
 echo "  ✅ Completed rapid start/stop cycles"
 # Test 2: Restart immediately after start
 echo ""
 echo "Test 2: Restart immediately after start..."
 crwl server start >/dev/null 2>&1
 sleep 3
 crwl server restart >/dev/null 2>&1
 sleep 5
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "  ❌ Health check failed after rapid restart"
    crwl server stop
    exit 1
 fi
 echo "  ✅ Rapid restart successful"
 # Test 3: Multiple status checks
 echo ""
 echo "Test 3: Multiple rapid status checks..."
 for i in {1..5}; do
    crwl server status >/dev/null 2>&1 || echo "  ⚠️  Status check $i failed"
 done
 echo "  ✅ Multiple status checks completed"
 # Test 4: Stop and immediate start
 echo ""
 echo "Test 4: Stop and immediate start..."
 crwl server stop >/dev/null 2>&1
 sleep 2
 crwl server start >/dev/null 2>&1
 sleep 5
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "  ❌ Health check failed after stop/start"
    crwl server stop
    exit 1
 fi
 echo "  ✅ Stop/immediate start successful"
 # Cleanup
 crwl server stop >/dev/null 2>&1
 echo ""
 echo "✅ Test passed: System handles rapid operations correctly"
--- a/deploy/docker/tests/cli/plan.md
+++ b/deploy/docker/tests/cli/plan.md
@@ -1,119 +0,0 @@
 E2E CLI Test Suite Plan                                                                                     │ │
 │ │                                                                                                             │ │
 │ │ Test Structure                                                                                              │ │
 │ │                                                                                                             │ │
 │ │ Create deploy/docker/tests/cli/ folder with individual test scripts organized by category.                  │ │
 │ │                                                                                                             │ │
 │ │ Test Categories                                                                                             │ │
 │ │                                                                                                             │ │
 │ │ 1. Basic Tests (deploy/docker/tests/cli/basic/)                                                             │ │
 │ │                                                                                                             │ │
 │ │ - test_01_start_default.sh - Start server with defaults (1 replica, port 11235)                             │ │
 │ │ - test_02_status.sh - Check server status                                                                   │ │
 │ │ - test_03_stop.sh - Stop server cleanly                                                                     │ │
 │ │ - test_04_start_custom_port.sh - Start with custom port (8080)                                              │ │
 │ │ - test_05_start_replicas.sh - Start with 3 replicas                                                         │ │
 │ │ - test_06_logs.sh - View logs (tail and follow)                                                             │ │
 │ │ - test_07_restart.sh - Restart server preserving config                                                     │ │
 │ │ - test_08_cleanup.sh - Force cleanup all resources                                                          │ │
 │ │                                                                                                             │ │
 │ │ 2. Advanced Tests (deploy/docker/tests/cli/advanced/)                                                       │ │
 │ │                                                                                                             │ │
 │ │ - test_01_scale_up.sh - Scale from 3 to 5 replicas                                                          │ │
 │ │ - test_02_scale_down.sh - Scale from 5 to 2 replicas                                                        │ │
 │ │ - test_03_mode_single.sh - Start in single mode explicitly                                                  │ │
 │ │ - test_04_mode_compose.sh - Start in compose mode with 3 replicas                                           │ │
 │ │ - test_05_custom_image.sh - Start with custom image tag                                                     │ │
 │ │ - test_06_env_file.sh - Start with custom env file                                                          │ │
 │ │ - test_07_stop_remove_volumes.sh - Stop and remove volumes                                                  │ │
 │ │ - test_08_restart_with_scale.sh - Restart and change replica count                                          │ │
 │ │                                                                                                             │ │
 │ │ 3. Resource Tests (deploy/docker/tests/cli/resource/)                                                       │ │
 │ │                                                                                                             │ │
 │ │ - test_01_memory_monitoring.sh - Monitor memory during crawls                                               │ │
 │ │ - test_02_cpu_stress.sh - CPU usage under concurrent load                                                   │ │
 │ │ - test_03_max_replicas.sh - Start with 10 replicas and stress test                                          │ │
 │ │ - test_04_cleanup_verification.sh - Verify all resources cleaned up                                         │ │
 │ │ - test_05_long_running.sh - Stability test (30 min runtime)                                                 │ │
 │ │                                                                                                             │ │
 │ │ 4. Dashboard UI Tests (deploy/docker/tests/cli/dashboard/)                                                  │ │
 │ │                                                                                                             │ │
 │ │ - test_01_dashboard_ui.py - Playwright test with screenshots                                                │ │
 │ │   - Start server with 3 replicas                                                                            │ │
 │ │   - Run demo_monitor_dashboard.py script                                                                    │ │
 │ │   - Use Playwright to:                                                                                      │ │
 │ │       - Take screenshot of main dashboard                                                                   │ │
 │ │     - Verify container filter buttons (All, C-1, C-2, C-3)                                                  │ │
 │ │     - Test WebSocket connection indicator                                                                   │ │
 │ │     - Verify timeline charts render                                                                         │ │
 │ │     - Test filtering functionality                                                                          │ │
 │ │     - Check all tabs (Requests, Browsers, Janitor, Errors, Stats)                                           │ │
 │ │                                                                                                             │ │
 │ │ 5. Edge Cases (deploy/docker/tests/cli/edge/)                                                               │ │
 │ │                                                                                                             │ │
 │ │ - test_01_already_running.sh - Try starting when already running                                            │ │
 │ │ - test_02_not_running.sh - Try stop/status when not running                                                 │ │
 │ │ - test_03_scale_single_mode.sh - Try scaling single container mode                                          │ │
 │ │ - test_04_invalid_port.sh - Invalid port numbers (0, -1, 99999)                                             │ │
 │ │ - test_05_invalid_replicas.sh - Invalid replica counts (0, -1, 101)                                         │ │
 │ │ - test_06_missing_env_file.sh - Non-existent env file                                                       │ │
 │ │ - test_07_port_in_use.sh - Port already occupied                                                            │ │
 │ │ - test_08_state_corruption.sh - Manually corrupt state file                                                 │ │
 │ │ - test_09_network_conflict.sh - Docker network name collision                                               │ │
 │ │ - test_10_rapid_operations.sh - Start/stop/restart in quick succession                                      │ │
 │ │                                                                                                             │ │
 │ │ Test Execution Plan                                                                                         │ │
 │ │                                                                                                             │ │
 │ │ Process:                                                                                                    │ │
 │ │                                                                                                             │ │
 │ │ 1. Create test file                                                                                         │ │
 │ │ 2. Run test                                                                                                 │ │
 │ │ 3. Verify results                                                                                           │ │
 │ │ 4. If fails → fix issue → re-test                                                                           │ │
 │ │ 5. Move to next test                                                                                        │ │
 │ │ 6. Clean up after each test to ensure clean state                                                           │ │
 │ │                                                                                                             │ │
 │ │ Common Test Structure:                                                                                      │ │
 │ │                                                                                                             │ │
 │ │ #!/bin/bash                                                                                                 │ │
 │ │ # Test: [Description]                                                                                       │ │
 │ │ # Expected: [What should happen]                                                                            │ │
 │ │                                                                                                             │ │
 │ │ source venv/bin/activate                                                                                    │ │
 │ │ set -e  # Exit on error                                                                                     │ │
 │ │                                                                                                             │ │
 │ │ echo "=== Test: [Name] ==="                                                                                 │ │
 │ │                                                                                                             │ │
 │ │ # Setup                                                                                                     │ │
 │ │ # ... test commands ...                                                                                     │ │
 │ │                                                                                                             │ │
 │ │ # Verification                                                                                              │ │
 │ │ # ... assertions ...                                                                                        │ │
 │ │                                                                                                             │ │
 │ │ # Cleanup                                                                                                   │ │
 │ │ crwl server stop || true                                                                                    │ │
 │ │                                                                                                             │ │
 │ │ echo "✓ Test passed"                                                                                        │ │
 │ │                                                                                                             │ │
 │ │ Dashboard Test Structure (Python):                                                                          │ │
 │ │                                                                                                             │ │
 │ │ # Activate venv first in calling script                                                                     │ │
 │ │ import asyncio                                                                                              │ │
 │ │ from playwright.async_api import async_playwright                                                           │ │
 │ │                                                                                                             │ │
 │ │ async def test_dashboard():                                                                                 │ │
 │ │     # Start server with 3 replicas                                                                          │ │
 │ │     # Run demo script in background                                                                         │ │
 │ │     # Launch Playwright                                                                                     │ │
 │ │     # Take screenshots                                                                                      │ │
 │ │     # Verify elements                                                                                       │ │
 │ │     # Cleanup                                                                                               │ │
 │ │                                                                                                             │ │
 │ │ Success Criteria:                                                                                           │ │
 │ │                                                                                                             │ │
 │ │ - All basic operations work correctly                                                                       │ │
 │ │ - Scaling operations function properly                                                                      │ │
 │ │ - Resource limits are respected                                                                             │ │
 │ │ - Dashboard UI is functional and responsive                                                                 │ │
 │ │ - Edge cases handled gracefully with proper error messages                                                  │ │
 │ │ - Clean resource cleanup verified
--- a/deploy/docker/tests/cli/resource/test_01_memory_monitoring.sh
+++ b/deploy/docker/tests/cli/resource/test_01_memory_monitoring.sh
@@ -1,63 +0,0 @@
 #!/bin/bash
 # Test: Monitor memory usage during crawl operations
 # Expected: Memory stats are accessible and reasonable
 set -e
 echo "=== Test: Memory Monitoring ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start server
 echo "Starting server..."
 crwl server start >/dev/null 2>&1
 sleep 5
 # Get baseline memory
 echo "Checking baseline memory..."
 BASELINE=$(curl -s http://localhost:11235/monitor/health | jq -r '.container.memory_percent' 2>/dev/null || echo "0")
 echo "Baseline memory: ${BASELINE}%"
 # Make several crawl requests
 echo ""
 echo "Making crawl requests to increase memory usage..."
 for i in {1..5}; do
    echo "  Request $i/5..."
    curl -s -X POST http://localhost:11235/crawl \
      -H "Content-Type: application/json" \
      -d "{\"urls\": [\"https://httpbin.org/html?req=$i\"], \"crawler_config\": {}}" > /dev/null || true
    sleep 1
 done
 # Check memory after requests
 echo ""
 echo "Checking memory after requests..."
 AFTER=$(curl -s http://localhost:11235/monitor/health | jq -r '.container.memory_percent' 2>/dev/null || echo "0")
 echo "Memory after requests: ${AFTER}%"
 # Get browser pool stats
 echo ""
 echo "Browser pool memory usage..."
 POOL_MEM=$(curl -s http://localhost:11235/monitor/browsers | jq -r '.summary.total_memory_mb' 2>/dev/null || echo "0")
 echo "Browser pool: ${POOL_MEM} MB"
 # Verify memory is within reasonable bounds (<80%)
 MEMORY_OK=$(echo "$AFTER < 80" | bc -l 2>/dev/null || echo "1")
 if [[ "$MEMORY_OK" != "1" ]]; then
    echo "⚠️  Warning: Memory usage is high: ${AFTER}%"
 fi
 # Cleanup
 echo ""
 echo "Cleaning up..."
 crwl server stop >/dev/null 2>&1
 echo ""
 echo "✅ Test passed: Memory monitoring functional"
 echo "   Baseline: ${BASELINE}%, After: ${AFTER}%, Pool: ${POOL_MEM} MB"
--- a/deploy/docker/tests/cli/resource/test_02_cpu_stress.sh
+++ b/deploy/docker/tests/cli/resource/test_02_cpu_stress.sh
@@ -1,61 +0,0 @@
 #!/bin/bash
 # Test: CPU usage under concurrent load
 # Expected: Server handles concurrent requests without errors
 set -e
 echo "=== Test: CPU Stress Test ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start server with 3 replicas for better load distribution
 echo "Starting server with 3 replicas..."
 crwl server start --replicas 3 >/dev/null 2>&1
 sleep 12
 # Get baseline CPU
 echo "Checking baseline container stats..."
 docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}" \
  --filter "name=crawl4ai" 2>/dev/null || echo "Unable to get container stats"
 # Send concurrent requests
 echo ""
 echo "Sending 10 concurrent requests..."
 for i in {1..10}; do
    curl -s -X POST http://localhost:11235/crawl \
      -H "Content-Type: application/json" \
      -d "{\"urls\": [\"https://httpbin.org/html?req=$i\"], \"crawler_config\": {}}" > /dev/null &
 done
 # Wait for all requests to complete
 echo "Waiting for requests to complete..."
 wait
 # Check stats after load
 echo ""
 echo "Container stats after load:"
 docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}" \
  --filter "name=crawl4ai" 2>/dev/null || echo "Unable to get container stats"
 # Verify health
 echo ""
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "❌ Health check failed after CPU stress"
    crwl server stop
    exit 1
 fi
 # Cleanup
 echo ""
 echo "Cleaning up..."
 crwl server stop >/dev/null 2>&1
 echo ""
 echo "✅ Test passed: Server handled concurrent load successfully"
--- a/deploy/docker/tests/cli/resource/test_03_max_replicas.sh
+++ b/deploy/docker/tests/cli/resource/test_03_max_replicas.sh
@@ -1,72 +0,0 @@
 #!/bin/bash
 # Test: Start with maximum replicas and stress test
 # Expected: Server handles max replicas (10) and distributes load
 set -e
 echo "=== Test: Maximum Replicas Stress Test ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start with 10 replicas (max recommended)
 echo "Starting server with 10 replicas..."
 echo "This may take some time..."
 crwl server start --replicas 10 >/dev/null 2>&1
 sleep 20
 # Verify status
 echo "Checking status..."
 STATUS=$(crwl server status)
 if ! echo "$STATUS" | grep -q "10"; then
    echo "❌ Failed to start 10 replicas"
    crwl server stop
    exit 1
 fi
 # Wait for container discovery
 echo ""
 echo "Waiting for container discovery..."
 sleep 10
 # Check containers
 CONTAINER_COUNT=$(curl -s http://localhost:11235/monitor/containers | jq -r '.count' 2>/dev/null || echo "0")
 echo "Discovered containers: $CONTAINER_COUNT"
 # Send burst of requests
 echo ""
 echo "Sending burst of 20 requests..."
 for i in {1..20}; do
    curl -s -X POST http://localhost:11235/crawl \
      -H "Content-Type: application/json" \
      -d "{\"urls\": [\"https://httpbin.org/html?req=$i\"], \"crawler_config\": {}}" > /dev/null &
 done
 wait
 # Check health after stress
 echo ""
 HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
 if [[ "$HEALTH" != "ok" ]]; then
    echo "❌ Health check failed after max replica stress"
    crwl server stop
    exit 1
 fi
 # Check endpoint stats
 echo ""
 echo "Endpoint statistics:"
 curl -s http://localhost:11235/monitor/endpoints/stats | jq '.' 2>/dev/null || echo "No stats available"
 # Cleanup
 echo ""
 echo "Cleaning up..."
 crwl server stop >/dev/null 2>&1
 echo ""
 echo "✅ Test passed: Successfully stress tested with 10 replicas"
--- a/deploy/docker/tests/cli/resource/test_04_cleanup_verification.sh
+++ b/deploy/docker/tests/cli/resource/test_04_cleanup_verification.sh
@@ -1,63 +0,0 @@
 #!/bin/bash
 # Test: Verify complete resource cleanup
 # Expected: All Docker resources are properly removed
 set -e
 echo "=== Test: Resource Cleanup Verification ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Start server to create resources
 echo "Starting server with 3 replicas..."
 crwl server start --replicas 3 >/dev/null 2>&1
 sleep 10
 # List resources before cleanup
 echo ""
 echo "Resources before cleanup:"
 echo "Containers:"
 docker ps --filter "name=crawl4ai" --format "  - {{.Names}}" 2>/dev/null || echo "  None"
 docker ps --filter "name=nginx" --format "  - {{.Names}}" 2>/dev/null || echo "  None"
 docker ps --filter "name=redis" --format "  - {{.Names}}" 2>/dev/null || echo "  None"
 echo ""
 echo "Networks:"
 docker network ls --filter "name=crawl4ai" --format "  - {{.Name}}" 2>/dev/null || echo "  None"
 # Cleanup
 echo ""
 echo "Performing cleanup..."
 crwl server cleanup --force >/dev/null 2>&1
 sleep 5
 # Verify cleanup
 echo ""
 echo "Verifying cleanup..."
 CONTAINERS=$(docker ps -a --filter "name=crawl4ai" --format "{{.Names}}" 2>/dev/null || echo "")
 if [[ -n "$CONTAINERS" ]]; then
    echo "❌ Found remaining crawl4ai containers: $CONTAINERS"
    exit 1
 fi
 NGINX=$(docker ps -a --filter "name=nginx" --format "{{.Names}}" 2>/dev/null || echo "")
 if [[ -n "$NGINX" ]]; then
    echo "⚠️  Warning: Nginx container still exists: $NGINX"
 fi
 REDIS=$(docker ps -a --filter "name=redis" --format "{{.Names}}" 2>/dev/null || echo "")
 if [[ -n "$REDIS" ]]; then
    echo "⚠️  Warning: Redis container still exists: $REDIS"
 fi
 # Verify port is free
 if curl -s http://localhost:11235/health > /dev/null 2>&1; then
    echo "❌ Port 11235 still in use after cleanup"
    exit 1
 fi
 echo ""
 echo "✅ Test passed: All Crawl4AI resources properly cleaned up"
--- a/deploy/docker/tests/cli/resource/test_05_long_running.sh
+++ b/deploy/docker/tests/cli/resource/test_05_long_running.sh
@@ -1,99 +0,0 @@
 #!/bin/bash
 # Test: Long-running stability test (5 minutes)
 # Expected: Server remains stable over extended period
 set -e
 echo "=== Test: Long-Running Stability (5 minutes) ==="
 echo ""
 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
 source "$PROJECT_ROOT/venv/bin/activate"
 # Cleanup
 crwl server stop 2>/dev/null || true
 sleep 2
 # Start server
 echo "Starting server with 2 replicas..."
 crwl server start --replicas 2 >/dev/null 2>&1
 sleep 10
 # Get start time
 START_TIME=$(date +%s)
 DURATION=300  # 5 minutes in seconds
 REQUEST_COUNT=0
 ERROR_COUNT=0
 echo ""
 echo "Running stability test for 5 minutes..."
 echo "Making periodic requests every 10 seconds..."
 echo ""
 while true; do
    CURRENT_TIME=$(date +%s)
    ELAPSED=$((CURRENT_TIME - START_TIME))
    if [[ $ELAPSED -ge $DURATION ]]; then
        break
    fi
    REMAINING=$((DURATION - ELAPSED))
    echo "[$ELAPSED/$DURATION seconds] Remaining: ${REMAINING}s, Requests: $REQUEST_COUNT, Errors: $ERROR_COUNT"
    # Make a request
    if curl -s -X POST http://localhost:11235/crawl \
        -H "Content-Type: application/json" \
        -d '{"urls": ["https://httpbin.org/html"], "crawler_config": {}}' > /dev/null 2>&1; then
        REQUEST_COUNT=$((REQUEST_COUNT + 1))
    else
        ERROR_COUNT=$((ERROR_COUNT + 1))
        echo "  ⚠️  Request failed"
    fi
    # Check health every 30 seconds
    if [[ $((ELAPSED % 30)) -eq 0 ]]; then
        HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
        if [[ "$HEALTH" != "ok" ]]; then
            echo "  ❌ Health check failed!"
            ERROR_COUNT=$((ERROR_COUNT + 1))
        fi
        # Get memory stats
        MEM=$(curl -s http://localhost:11235/monitor/health | jq -r '.container.memory_percent' 2>/dev/null || echo "N/A")
        echo "  Memory: ${MEM}%"
    fi
    sleep 10
 done
 echo ""
 echo "Test duration completed!"
 echo "Total requests: $REQUEST_COUNT"
 echo "Total errors: $ERROR_COUNT"
 # Get final stats
 echo ""
 echo "Final statistics:"
 curl -s http://localhost:11235/monitor/endpoints/stats | jq '.' 2>/dev/null || echo "No stats available"
 # Verify error rate is acceptable (<10%)
 ERROR_RATE=$(echo "scale=2; $ERROR_COUNT * 100 / $REQUEST_COUNT" | bc -l 2>/dev/null || echo "0")
 echo ""
 echo "Error rate: ${ERROR_RATE}%"
 # Cleanup
 echo ""
 echo "Cleaning up..."
 crwl server stop >/dev/null 2>&1
 # Check error rate
 ERROR_OK=$(echo "$ERROR_RATE < 10" | bc -l 2>/dev/null || echo "1")
 if [[ "$ERROR_OK" != "1" ]]; then
    echo "❌ Error rate too high: ${ERROR_RATE}%"
    exit 1
 fi
 echo ""
 echo "✅ Test passed: Server remained stable over 5 minutes"
 echo "   Requests: $REQUEST_COUNT, Errors: $ERROR_COUNT, Error rate: ${ERROR_RATE}%"
--- a/deploy/docker/tests/cli/run_tests.sh
+++ b/deploy/docker/tests/cli/run_tests.sh
@@ -1,200 +0,0 @@
 #!/bin/bash
 # Master Test Runner for Crawl4AI CLI E2E Tests
 # Usage: ./run_tests.sh [category] [test_number]
 #   category: basic|advanced|resource|dashboard|edge|all (default: all)
 #   test_number: specific test number to run (optional)
 set -e
 # Color codes for output
 RED='\033[0;31m'
 GREEN='\033[0;32m'
 YELLOW='\033[1;33m'
 BLUE='\033[0;34m'
 NC='\033[0m' # No Color
 # Test counters
 TOTAL_TESTS=0
 PASSED_TESTS=0
 FAILED_TESTS=0
 SKIPPED_TESTS=0
 # Get script directory
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # Print header
 print_header() {
    echo ""
    echo "=========================================="
    echo "$1"
    echo "=========================================="
    echo ""
 }
 # Print test result
 print_result() {
    local test_name=$1
    local result=$2
    if [[ "$result" == "PASS" ]]; then
        echo -e "${GREEN}✅ PASS${NC}: $test_name"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    elif [[ "$result" == "FAIL" ]]; then
        echo -e "${RED}❌ FAIL${NC}: $test_name"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    elif [[ "$result" == "SKIP" ]]; then
        echo -e "${YELLOW}⏭️  SKIP${NC}: $test_name"
        SKIPPED_TESTS=$((SKIPPED_TESTS + 1))
    fi
 }
 # Run a single test
 run_test() {
    local test_path=$1
    local test_name=$(basename "$test_path")
    echo ""
    echo -e "${BLUE}Running:${NC} $test_name"
    echo "----------------------------------------"
    TOTAL_TESTS=$((TOTAL_TESTS + 1))
    if bash "$test_path"; then
        print_result "$test_name" "PASS"
        return 0
    else
        print_result "$test_name" "FAIL"
        return 1
    fi
 }
 # Run Python test
 run_python_test() {
    local test_path=$1
    local test_name=$(basename "$test_path")
    echo ""
    echo -e "${BLUE}Running:${NC} $test_name"
    echo "----------------------------------------"
    TOTAL_TESTS=$((TOTAL_TESTS + 1))
    if python "$test_path"; then
        print_result "$test_name" "PASS"
        return 0
    else
        print_result "$test_name" "FAIL"
        return 1
    fi
 }
 # Run tests in a category
 run_category() {
    local category=$1
    local test_number=$2
    local category_dir="$SCRIPT_DIR/$category"
    if [[ ! -d "$category_dir" ]]; then
        echo -e "${RED}Error:${NC} Category '$category' not found"
        return 1
    fi
    print_header "Running $category tests"
    if [[ -n "$test_number" ]]; then
        # Run specific test
        local test_file=$(find "$category_dir" -name "*${test_number}*.sh" | head -n 1)
        if [[ -z "$test_file" ]]; then
            echo -e "${RED}Error:${NC} Test $test_number not found in $category"
            return 1
        fi
        run_test "$test_file"
    else
        # Run all tests in category
        if [[ "$category" == "dashboard" ]]; then
            # Dashboard tests are Python
            for test_file in "$category_dir"/*.py; do
                [[ -f "$test_file" ]] || continue
                run_python_test "$test_file" || true
            done
        else
            # Shell script tests
            for test_file in "$category_dir"/*.sh; do
                [[ -f "$test_file" ]] || continue
                run_test "$test_file" || true
            done
        fi
    fi
 }
 # Print summary
 print_summary() {
    echo ""
    echo "=========================================="
    echo "Test Summary"
    echo "=========================================="
    echo -e "Total:   $TOTAL_TESTS"
    echo -e "${GREEN}Passed:  $PASSED_TESTS${NC}"
    echo -e "${RED}Failed:  $FAILED_TESTS${NC}"
    echo -e "${YELLOW}Skipped: $SKIPPED_TESTS${NC}"
    echo ""
    if [[ $FAILED_TESTS -eq 0 ]]; then
        echo -e "${GREEN}✅ All tests passed!${NC}"
        return 0
    else
        echo -e "${RED}❌ Some tests failed${NC}"
        return 1
    fi
 }
 # Main execution
 main() {
    local category=${1:-all}
    local test_number=$2
    # Activate virtual environment
    if [[ -f "venv/bin/activate" ]]; then
        source venv/bin/activate
    else
        echo -e "${YELLOW}Warning:${NC} venv not found, some tests may fail"
    fi
    print_header "Crawl4AI CLI E2E Test Suite"
    if [[ "$category" == "all" ]]; then
        # Run all categories
        for cat in basic advanced resource edge; do
            run_category "$cat" || true
        done
        # Dashboard tests separately (can be slow)
        echo ""
        echo -e "${YELLOW}Note:${NC} Dashboard tests can be run separately with: ./run_tests.sh dashboard"
    else
        run_category "$category" "$test_number"
    fi
    print_summary
 }
 # Show usage
 if [[ "$1" == "-h" || "$1" == "--help" ]]; then
    echo "Usage: $0 [category] [test_number]"
    echo ""
    echo "Categories:"
    echo "  basic      - Basic CLI operations (8 tests)"
    echo "  advanced   - Advanced features (8 tests)"
    echo "  resource   - Resource monitoring and stress tests (5 tests)"
    echo "  dashboard  - Dashboard UI tests with Playwright (1 test)"
    echo "  edge       - Edge cases and error handling (10 tests)"
    echo "  all        - Run all tests except dashboard (default)"
    echo ""
    echo "Examples:"
    echo "  $0                    # Run all tests"
    echo "  $0 basic              # Run all basic tests"
    echo "  $0 basic 01           # Run test_01 from basic"
    echo "  $0 dashboard          # Run dashboard UI test"
    exit 0
 fi
 main "$@"
--- a/deploy/docker/tests/codebase_test/test_1_basic.py
+++ b/deploy/docker/tests/codebase_test/test_1_basic.py
@@ -1,138 +0,0 @@
 #!/usr/bin/env python3
 """
 Test 1: Basic Container Health + Single Endpoint
 - Starts container
 - Hits /health endpoint 10 times
 - Reports success rate and basic latency
 """
 import asyncio
 import time
 import docker
 import httpx
 # Config
 IMAGE = "crawl4ai-local:latest"
 CONTAINER_NAME = "crawl4ai-test"
 PORT = 11235
 REQUESTS = 10
 async def test_endpoint(url: str, count: int):
    """Hit endpoint multiple times, return stats."""
    results = []
    async with httpx.AsyncClient(timeout=30.0) as client:
        for i in range(count):
            start = time.time()
            try:
                resp = await client.get(url)
                elapsed = (time.time() - start) * 1000  # ms
                results.append({
                    "success": resp.status_code == 200,
                    "latency_ms": elapsed,
                    "status": resp.status_code
                })
                print(f"  [{i+1}/{count}] ✓ {resp.status_code} - {elapsed:.0f}ms")
            except Exception as e:
                results.append({
                    "success": False,
                    "latency_ms": None,
                    "error": str(e)
                })
                print(f"  [{i+1}/{count}] ✗ Error: {e}")
    return results
 def start_container(client, image: str, name: str, port: int):
    """Start container, return container object."""
    # Clean up existing
    try:
        old = client.containers.get(name)
        print(f"🧹 Stopping existing container '{name}'...")
        old.stop()
        old.remove()
    except docker.errors.NotFound:
        pass
    print(f"🚀 Starting container '{name}' from image '{image}'...")
    container = client.containers.run(
        image,
        name=name,
        ports={f"{port}/tcp": port},
        detach=True,
        shm_size="1g",
        environment={"PYTHON_ENV": "production"}
    )
    # Wait for health
    print(f"⏳ Waiting for container to be healthy...")
    for _ in range(30):  # 30s timeout
        time.sleep(1)
        container.reload()
        if container.status == "running":
            try:
                # Quick health check
                import requests
                resp = requests.get(f"http://localhost:{port}/health", timeout=2)
                if resp.status_code == 200:
                    print(f"✅ Container healthy!")
                    return container
            except:
                pass
    raise TimeoutError("Container failed to start")
 def stop_container(container):
    """Stop and remove container."""
    print(f"🛑 Stopping container...")
    container.stop()
    container.remove()
    print(f"✅ Container removed")
 async def main():
    print("="*60)
    print("TEST 1: Basic Container Health + Single Endpoint")
    print("="*60)
    client = docker.from_env()
    container = None
    try:
        # Start container
        container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
        # Test /health endpoint
        print(f"\n📊 Testing /health endpoint ({REQUESTS} requests)...")
        url = f"http://localhost:{PORT}/health"
        results = await test_endpoint(url, REQUESTS)
        # Calculate stats
        successes = sum(1 for r in results if r["success"])
        success_rate = (successes / len(results)) * 100
        latencies = [r["latency_ms"] for r in results if r["latency_ms"] is not None]
        avg_latency = sum(latencies) / len(latencies) if latencies else 0
        # Print results
        print(f"\n{'='*60}")
        print(f"RESULTS:")
        print(f"  Success Rate: {success_rate:.1f}% ({successes}/{len(results)})")
        print(f"  Avg Latency:  {avg_latency:.0f}ms")
        if latencies:
            print(f"  Min Latency:  {min(latencies):.0f}ms")
            print(f"  Max Latency:  {max(latencies):.0f}ms")
        print(f"{'='*60}")
        # Pass/Fail
        if success_rate >= 100:
            print(f"✅ TEST PASSED")
            return 0
        else:
            print(f"❌ TEST FAILED (expected 100% success rate)")
            return 1
    except Exception as e:
        print(f"\n❌ TEST ERROR: {e}")
        return 1
    finally:
        if container:
            stop_container(container)
 if __name__ == "__main__":
    exit_code = asyncio.run(main())
    exit(exit_code)
--- a/deploy/docker/tests/codebase_test/test_2_memory.py
+++ b/deploy/docker/tests/codebase_test/test_2_memory.py
@@ -1,205 +0,0 @@
 #!/usr/bin/env python3
 """
 Test 2: Docker Stats Monitoring
 - Extends Test 1 with real-time container stats
 - Monitors memory % and CPU during requests
 - Reports baseline, peak, and final memory
 """
 import asyncio
 import time
 import docker
 import httpx
 from threading import Thread, Event
 # Config
 IMAGE = "crawl4ai-local:latest"
 CONTAINER_NAME = "crawl4ai-test"
 PORT = 11235
 REQUESTS = 20  # More requests to see memory usage
 # Stats tracking
 stats_history = []
 stop_monitoring = Event()
 def monitor_stats(container):
    """Background thread to collect container stats."""
    for stat in container.stats(decode=True, stream=True):
        if stop_monitoring.is_set():
            break
        try:
            # Extract memory stats
            mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)  # MB
            mem_limit = stat['memory_stats'].get('limit', 1) / (1024 * 1024)
            mem_percent = (mem_usage / mem_limit * 100) if mem_limit > 0 else 0
            # Extract CPU stats (handle missing fields on Mac)
            cpu_percent = 0
            try:
                cpu_delta = stat['cpu_stats']['cpu_usage']['total_usage'] - \
                           stat['precpu_stats']['cpu_usage']['total_usage']
                system_delta = stat['cpu_stats'].get('system_cpu_usage', 0) - \
                              stat['precpu_stats'].get('system_cpu_usage', 0)
                if system_delta > 0:
                    num_cpus = stat['cpu_stats'].get('online_cpus', 1)
                    cpu_percent = (cpu_delta / system_delta * num_cpus * 100.0)
            except (KeyError, ZeroDivisionError):
                pass
            stats_history.append({
                'timestamp': time.time(),
                'memory_mb': mem_usage,
                'memory_percent': mem_percent,
                'cpu_percent': cpu_percent
            })
        except Exception as e:
            # Skip malformed stats
            pass
        time.sleep(0.5)  # Sample every 500ms
 async def test_endpoint(url: str, count: int):
    """Hit endpoint, return stats."""
    results = []
    async with httpx.AsyncClient(timeout=30.0) as client:
        for i in range(count):
            start = time.time()
            try:
                resp = await client.get(url)
                elapsed = (time.time() - start) * 1000
                results.append({
                    "success": resp.status_code == 200,
                    "latency_ms": elapsed,
                })
                if (i + 1) % 5 == 0:  # Print every 5 requests
                    print(f"  [{i+1}/{count}] ✓ {resp.status_code} - {elapsed:.0f}ms")
            except Exception as e:
                results.append({"success": False, "error": str(e)})
                print(f"  [{i+1}/{count}] ✗ Error: {e}")
    return results
 def start_container(client, image: str, name: str, port: int):
    """Start container."""
    try:
        old = client.containers.get(name)
        print(f"🧹 Stopping existing container '{name}'...")
        old.stop()
        old.remove()
    except docker.errors.NotFound:
        pass
    print(f"🚀 Starting container '{name}'...")
    container = client.containers.run(
        image,
        name=name,
        ports={f"{port}/tcp": port},
        detach=True,
        shm_size="1g",
        mem_limit="4g",  # Set explicit memory limit
    )
    print(f"⏳ Waiting for health...")
    for _ in range(30):
        time.sleep(1)
        container.reload()
        if container.status == "running":
            try:
                import requests
                resp = requests.get(f"http://localhost:{port}/health", timeout=2)
                if resp.status_code == 200:
                    print(f"✅ Container healthy!")
                    return container
            except:
                pass
    raise TimeoutError("Container failed to start")
 def stop_container(container):
    """Stop container."""
    print(f"🛑 Stopping container...")
    container.stop()
    container.remove()
 async def main():
    print("="*60)
    print("TEST 2: Docker Stats Monitoring")
    print("="*60)
    client = docker.from_env()
    container = None
    monitor_thread = None
    try:
        # Start container
        container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
        # Start stats monitoring in background
        print(f"\n📊 Starting stats monitor...")
        stop_monitoring.clear()
        stats_history.clear()
        monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
        monitor_thread.start()
        # Wait a bit for baseline
        await asyncio.sleep(2)
        baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
        print(f"📏 Baseline memory: {baseline_mem:.1f} MB")
        # Test /health endpoint
        print(f"\n🔄 Running {REQUESTS} requests to /health...")
        url = f"http://localhost:{PORT}/health"
        results = await test_endpoint(url, REQUESTS)
        # Wait a bit to capture peak
        await asyncio.sleep(1)
        # Stop monitoring
        stop_monitoring.set()
        if monitor_thread:
            monitor_thread.join(timeout=2)
        # Calculate stats
        successes = sum(1 for r in results if r.get("success"))
        success_rate = (successes / len(results)) * 100
        latencies = [r["latency_ms"] for r in results if "latency_ms" in r]
        avg_latency = sum(latencies) / len(latencies) if latencies else 0
        # Memory stats
        memory_samples = [s['memory_mb'] for s in stats_history]
        peak_mem = max(memory_samples) if memory_samples else 0
        final_mem = memory_samples[-1] if memory_samples else 0
        mem_delta = final_mem - baseline_mem
        # Print results
        print(f"\n{'='*60}")
        print(f"RESULTS:")
        print(f"  Success Rate: {success_rate:.1f}% ({successes}/{len(results)})")
        print(f"  Avg Latency:  {avg_latency:.0f}ms")
        print(f"\n  Memory Stats:")
        print(f"    Baseline: {baseline_mem:.1f} MB")
        print(f"    Peak:     {peak_mem:.1f} MB")
        print(f"    Final:    {final_mem:.1f} MB")
        print(f"    Delta:    {mem_delta:+.1f} MB")
        print(f"{'='*60}")
        # Pass/Fail
        if success_rate >= 100 and mem_delta < 100:  # No significant memory growth
            print(f"✅ TEST PASSED")
            return 0
        else:
            if success_rate < 100:
                print(f"❌ TEST FAILED (success rate < 100%)")
            if mem_delta >= 100:
                print(f"⚠️  WARNING: Memory grew by {mem_delta:.1f} MB")
            return 1
    except Exception as e:
        print(f"\n❌ TEST ERROR: {e}")
        return 1
    finally:
        stop_monitoring.set()
        if container:
            stop_container(container)
 if __name__ == "__main__":
    exit_code = asyncio.run(main())
    exit(exit_code)
--- a/deploy/docker/tests/codebase_test/test_3_pool.py
+++ b/deploy/docker/tests/codebase_test/test_3_pool.py
@@ -1,229 +0,0 @@
 #!/usr/bin/env python3
 """
 Test 3: Pool Validation - Permanent Browser Reuse
 - Tests /html endpoint (should use permanent browser)
 - Monitors container logs for pool hit markers
 - Validates browser reuse rate
 - Checks memory after browser creation
 """
 import asyncio
 import time
 import docker
 import httpx
 from threading import Thread, Event
 # Config
 IMAGE = "crawl4ai-local:latest"
 CONTAINER_NAME = "crawl4ai-test"
 PORT = 11235
 REQUESTS = 30
 # Stats tracking
 stats_history = []
 stop_monitoring = Event()
 def monitor_stats(container):
    """Background stats collector."""
    for stat in container.stats(decode=True, stream=True):
        if stop_monitoring.is_set():
            break
        try:
            mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
            stats_history.append({
                'timestamp': time.time(),
                'memory_mb': mem_usage,
            })
        except:
            pass
        time.sleep(0.5)
 def count_log_markers(container):
    """Extract pool usage markers from logs."""
    logs = container.logs().decode('utf-8')
    permanent_hits = logs.count("🔥 Using permanent browser")
    hot_hits = logs.count("♨️  Using hot pool browser")
    cold_hits = logs.count("❄️  Using cold pool browser")
    new_created = logs.count("🆕 Creating new browser")
    return {
        'permanent_hits': permanent_hits,
        'hot_hits': hot_hits,
        'cold_hits': cold_hits,
        'new_created': new_created,
        'total_hits': permanent_hits + hot_hits + cold_hits
    }
 async def test_endpoint(url: str, count: int):
    """Hit endpoint multiple times."""
    results = []
    async with httpx.AsyncClient(timeout=60.0) as client:
        for i in range(count):
            start = time.time()
            try:
                resp = await client.post(url, json={"url": "https://httpbin.org/html"})
                elapsed = (time.time() - start) * 1000
                results.append({
                    "success": resp.status_code == 200,
                    "latency_ms": elapsed,
                })
                if (i + 1) % 10 == 0:
                    print(f"  [{i+1}/{count}] ✓ {resp.status_code} - {elapsed:.0f}ms")
            except Exception as e:
                results.append({"success": False, "error": str(e)})
                print(f"  [{i+1}/{count}] ✗ Error: {e}")
    return results
 def start_container(client, image: str, name: str, port: int):
    """Start container."""
    try:
        old = client.containers.get(name)
        print(f"🧹 Stopping existing container...")
        old.stop()
        old.remove()
    except docker.errors.NotFound:
        pass
    print(f"🚀 Starting container...")
    container = client.containers.run(
        image,
        name=name,
        ports={f"{port}/tcp": port},
        detach=True,
        shm_size="1g",
        mem_limit="4g",
    )
    print(f"⏳ Waiting for health...")
    for _ in range(30):
        time.sleep(1)
        container.reload()
        if container.status == "running":
            try:
                import requests
                resp = requests.get(f"http://localhost:{port}/health", timeout=2)
                if resp.status_code == 200:
                    print(f"✅ Container healthy!")
                    return container
            except:
                pass
    raise TimeoutError("Container failed to start")
 def stop_container(container):
    """Stop container."""
    print(f"🛑 Stopping container...")
    container.stop()
    container.remove()
 async def main():
    print("="*60)
    print("TEST 3: Pool Validation - Permanent Browser Reuse")
    print("="*60)
    client = docker.from_env()
    container = None
    monitor_thread = None
    try:
        # Start container
        container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
        # Wait for permanent browser initialization
        print(f"\n⏳ Waiting for permanent browser init (3s)...")
        await asyncio.sleep(3)
        # Start stats monitoring
        print(f"📊 Starting stats monitor...")
        stop_monitoring.clear()
        stats_history.clear()
        monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
        monitor_thread.start()
        await asyncio.sleep(1)
        baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
        print(f"📏 Baseline (with permanent browser): {baseline_mem:.1f} MB")
        # Test /html endpoint (uses permanent browser for default config)
        print(f"\n🔄 Running {REQUESTS} requests to /html...")
        url = f"http://localhost:{PORT}/html"
        results = await test_endpoint(url, REQUESTS)
        # Wait a bit
        await asyncio.sleep(1)
        # Stop monitoring
        stop_monitoring.set()
        if monitor_thread:
            monitor_thread.join(timeout=2)
        # Analyze logs for pool markers
        print(f"\n📋 Analyzing pool usage...")
        pool_stats = count_log_markers(container)
        # Calculate request stats
        successes = sum(1 for r in results if r.get("success"))
        success_rate = (successes / len(results)) * 100
        latencies = [r["latency_ms"] for r in results if "latency_ms" in r]
        avg_latency = sum(latencies) / len(latencies) if latencies else 0
        # Memory stats
        memory_samples = [s['memory_mb'] for s in stats_history]
        peak_mem = max(memory_samples) if memory_samples else 0
        final_mem = memory_samples[-1] if memory_samples else 0
        mem_delta = final_mem - baseline_mem
        # Calculate reuse rate
        total_requests = len(results)
        total_pool_hits = pool_stats['total_hits']
        reuse_rate = (total_pool_hits / total_requests * 100) if total_requests > 0 else 0
        # Print results
        print(f"\n{'='*60}")
        print(f"RESULTS:")
        print(f"  Success Rate: {success_rate:.1f}% ({successes}/{len(results)})")
        print(f"  Avg Latency:  {avg_latency:.0f}ms")
        print(f"\n  Pool Stats:")
        print(f"    🔥 Permanent Hits: {pool_stats['permanent_hits']}")
        print(f"    ♨️  Hot Pool Hits:   {pool_stats['hot_hits']}")
        print(f"    ❄️  Cold Pool Hits:  {pool_stats['cold_hits']}")
        print(f"    🆕 New Created:    {pool_stats['new_created']}")
        print(f"    📊 Reuse Rate:     {reuse_rate:.1f}%")
        print(f"\n  Memory Stats:")
        print(f"    Baseline: {baseline_mem:.1f} MB")
        print(f"    Peak:     {peak_mem:.1f} MB")
        print(f"    Final:    {final_mem:.1f} MB")
        print(f"    Delta:    {mem_delta:+.1f} MB")
        print(f"{'='*60}")
        # Pass/Fail
        passed = True
        if success_rate < 100:
            print(f"❌ FAIL: Success rate {success_rate:.1f}% < 100%")
            passed = False
        if reuse_rate < 80:
            print(f"❌ FAIL: Reuse rate {reuse_rate:.1f}% < 80% (expected high permanent browser usage)")
            passed = False
        if pool_stats['permanent_hits'] < (total_requests * 0.8):
            print(f"⚠️  WARNING: Only {pool_stats['permanent_hits']} permanent hits out of {total_requests} requests")
        if mem_delta > 200:
            print(f"⚠️  WARNING: Memory grew by {mem_delta:.1f} MB (possible browser leak)")
        if passed:
            print(f"✅ TEST PASSED")
            return 0
        else:
            return 1
    except Exception as e:
        print(f"\n❌ TEST ERROR: {e}")
        import traceback
        traceback.print_exc()
        return 1
    finally:
        stop_monitoring.set()
        if container:
            stop_container(container)
 if __name__ == "__main__":
    exit_code = asyncio.run(main())
    exit(exit_code)
--- a/deploy/docker/tests/codebase_test/test_4_concurrent.py
+++ b/deploy/docker/tests/codebase_test/test_4_concurrent.py
@@ -1,236 +0,0 @@
 #!/usr/bin/env python3
 """
 Test 4: Concurrent Load Testing
 - Tests pool under concurrent load
 - Escalates: 10 → 50 → 100 concurrent requests
 - Validates latency distribution (P50, P95, P99)
 - Monitors memory stability
 """
 import asyncio
 import time
 import docker
 import httpx
 from threading import Thread, Event
 from collections import defaultdict
 # Config
 IMAGE = "crawl4ai-local:latest"
 CONTAINER_NAME = "crawl4ai-test"
 PORT = 11235
 LOAD_LEVELS = [
    {"name": "Light", "concurrent": 10, "requests": 20},
    {"name": "Medium", "concurrent": 50, "requests": 100},
    {"name": "Heavy", "concurrent": 100, "requests": 200},
 ]
 # Stats
 stats_history = []
 stop_monitoring = Event()
 def monitor_stats(container):
    """Background stats collector."""
    for stat in container.stats(decode=True, stream=True):
        if stop_monitoring.is_set():
            break
        try:
            mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
            stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage})
        except:
            pass
        time.sleep(0.5)
 def count_log_markers(container):
    """Extract pool markers."""
    logs = container.logs().decode('utf-8')
    return {
        'permanent': logs.count("🔥 Using permanent browser"),
        'hot': logs.count("♨️  Using hot pool browser"),
        'cold': logs.count("❄️  Using cold pool browser"),
        'new': logs.count("🆕 Creating new browser"),
    }
 async def hit_endpoint(client, url, payload, semaphore):
    """Single request with concurrency control."""
    async with semaphore:
        start = time.time()
        try:
            resp = await client.post(url, json=payload, timeout=60.0)
            elapsed = (time.time() - start) * 1000
            return {"success": resp.status_code == 200, "latency_ms": elapsed}
        except Exception as e:
            return {"success": False, "error": str(e)}
 async def run_concurrent_test(url, payload, concurrent, total_requests):
    """Run concurrent requests."""
    semaphore = asyncio.Semaphore(concurrent)
    async with httpx.AsyncClient() as client:
        tasks = [hit_endpoint(client, url, payload, semaphore) for _ in range(total_requests)]
        results = await asyncio.gather(*tasks)
    return results
 def calculate_percentiles(latencies):
    """Calculate P50, P95, P99."""
    if not latencies:
        return 0, 0, 0
    sorted_lat = sorted(latencies)
    n = len(sorted_lat)
    return (
        sorted_lat[int(n * 0.50)],
        sorted_lat[int(n * 0.95)],
        sorted_lat[int(n * 0.99)],
    )
 def start_container(client, image, name, port):
    """Start container."""
    try:
        old = client.containers.get(name)
        print(f"🧹 Stopping existing container...")
        old.stop()
        old.remove()
    except docker.errors.NotFound:
        pass
    print(f"🚀 Starting container...")
    container = client.containers.run(
        image, name=name, ports={f"{port}/tcp": port},
        detach=True, shm_size="1g", mem_limit="4g",
    )
    print(f"⏳ Waiting for health...")
    for _ in range(30):
        time.sleep(1)
        container.reload()
        if container.status == "running":
            try:
                import requests
                if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200:
                    print(f"✅ Container healthy!")
                    return container
            except:
                pass
    raise TimeoutError("Container failed to start")
 async def main():
    print("="*60)
    print("TEST 4: Concurrent Load Testing")
    print("="*60)
    client = docker.from_env()
    container = None
    monitor_thread = None
    try:
        container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
        print(f"\n⏳ Waiting for permanent browser init (3s)...")
        await asyncio.sleep(3)
        # Start monitoring
        stop_monitoring.clear()
        stats_history.clear()
        monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
        monitor_thread.start()
        await asyncio.sleep(1)
        baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
        print(f"📏 Baseline: {baseline_mem:.1f} MB\n")
        url = f"http://localhost:{PORT}/html"
        payload = {"url": "https://httpbin.org/html"}
        all_results = []
        level_stats = []
        # Run load levels
        for level in LOAD_LEVELS:
            print(f"{'='*60}")
            print(f"🔄 {level['name']} Load: {level['concurrent']} concurrent, {level['requests']} total")
            print(f"{'='*60}")
            start_time = time.time()
            results = await run_concurrent_test(url, payload, level['concurrent'], level['requests'])
            duration = time.time() - start_time
            successes = sum(1 for r in results if r.get("success"))
            success_rate = (successes / len(results)) * 100
            latencies = [r["latency_ms"] for r in results if "latency_ms" in r]
            p50, p95, p99 = calculate_percentiles(latencies)
            avg_lat = sum(latencies) / len(latencies) if latencies else 0
            print(f"  Duration:     {duration:.1f}s")
            print(f"  Success:      {success_rate:.1f}% ({successes}/{len(results)})")
            print(f"  Avg Latency:  {avg_lat:.0f}ms")
            print(f"  P50/P95/P99:  {p50:.0f}ms / {p95:.0f}ms / {p99:.0f}ms")
            level_stats.append({
                'name': level['name'],
                'concurrent': level['concurrent'],
                'success_rate': success_rate,
                'avg_latency': avg_lat,
                'p50': p50, 'p95': p95, 'p99': p99,
            })
            all_results.extend(results)
            await asyncio.sleep(2)  # Cool down between levels
        # Stop monitoring
        await asyncio.sleep(1)
        stop_monitoring.set()
        if monitor_thread:
            monitor_thread.join(timeout=2)
        # Final stats
        pool_stats = count_log_markers(container)
        memory_samples = [s['memory_mb'] for s in stats_history]
        peak_mem = max(memory_samples) if memory_samples else 0
        final_mem = memory_samples[-1] if memory_samples else 0
        print(f"\n{'='*60}")
        print(f"FINAL RESULTS:")
        print(f"{'='*60}")
        print(f"  Total Requests: {len(all_results)}")
        print(f"\n  Pool Utilization:")
        print(f"    🔥 Permanent: {pool_stats['permanent']}")
        print(f"    ♨️  Hot:       {pool_stats['hot']}")
        print(f"    ❄️  Cold:      {pool_stats['cold']}")
        print(f"    🆕 New:       {pool_stats['new']}")
        print(f"\n  Memory:")
        print(f"    Baseline: {baseline_mem:.1f} MB")
        print(f"    Peak:     {peak_mem:.1f} MB")
        print(f"    Final:    {final_mem:.1f} MB")
        print(f"    Delta:    {final_mem - baseline_mem:+.1f} MB")
        print(f"{'='*60}")
        # Pass/Fail
        passed = True
        for ls in level_stats:
            if ls['success_rate'] < 99:
                print(f"❌ FAIL: {ls['name']} success rate {ls['success_rate']:.1f}% < 99%")
                passed = False
            if ls['p99'] > 10000:  # 10s threshold
                print(f"⚠️  WARNING: {ls['name']} P99 latency {ls['p99']:.0f}ms very high")
        if final_mem - baseline_mem > 300:
            print(f"⚠️  WARNING: Memory grew {final_mem - baseline_mem:.1f} MB")
        if passed:
            print(f"✅ TEST PASSED")
            return 0
        else:
            return 1
    except Exception as e:
        print(f"\n❌ TEST ERROR: {e}")
        import traceback
        traceback.print_exc()
        return 1
    finally:
        stop_monitoring.set()
        if container:
            print(f"🛑 Stopping container...")
            container.stop()
            container.remove()
 if __name__ == "__main__":
    exit_code = asyncio.run(main())
    exit(exit_code)
--- a/deploy/docker/tests/codebase_test/test_5_pool_stress.py
+++ b/deploy/docker/tests/codebase_test/test_5_pool_stress.py
@@ -1,267 +0,0 @@
 #!/usr/bin/env python3
 """
 Test 5: Pool Stress - Mixed Configs
 - Tests hot/cold pool with different browser configs
 - Uses different viewports to create config variants
 - Validates cold → hot promotion after 3 uses
 - Monitors pool tier distribution
 """
 import asyncio
 import time
 import docker
 import httpx
 from threading import Thread, Event
 import random
 # Config
 IMAGE = "crawl4ai-local:latest"
 CONTAINER_NAME = "crawl4ai-test"
 PORT = 11235
 REQUESTS_PER_CONFIG = 5  # 5 requests per config variant
 # Different viewport configs to test pool tiers
 VIEWPORT_CONFIGS = [
    None,  # Default (permanent browser)
    {"width": 1920, "height": 1080},  # Desktop
    {"width": 1024, "height": 768},   # Tablet
    {"width": 375, "height": 667},    # Mobile
 ]
 # Stats
 stats_history = []
 stop_monitoring = Event()
 def monitor_stats(container):
    """Background stats collector."""
    for stat in container.stats(decode=True, stream=True):
        if stop_monitoring.is_set():
            break
        try:
            mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
            stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage})
        except:
            pass
        time.sleep(0.5)
 def analyze_pool_logs(container):
    """Extract detailed pool stats from logs."""
    logs = container.logs().decode('utf-8')
    permanent = logs.count("🔥 Using permanent browser")
    hot = logs.count("♨️  Using hot pool browser")
    cold = logs.count("❄️  Using cold pool browser")
    new = logs.count("🆕 Creating new browser")
    promotions = logs.count("⬆️  Promoting to hot pool")
    return {
        'permanent': permanent,
        'hot': hot,
        'cold': cold,
        'new': new,
        'promotions': promotions,
        'total': permanent + hot + cold
    }
 async def crawl_with_viewport(client, url, viewport):
    """Single request with specific viewport."""
    payload = {
        "urls": ["https://httpbin.org/html"],
        "browser_config": {},
        "crawler_config": {}
    }
    # Add viewport if specified
    if viewport:
        payload["browser_config"] = {
            "type": "BrowserConfig",
            "params": {
                "viewport": {"type": "dict", "value": viewport},
                "headless": True,
                "text_mode": True,
                "extra_args": [
                    "--no-sandbox",
                    "--disable-dev-shm-usage",
                    "--disable-gpu",
                    "--disable-software-rasterizer",
                    "--disable-web-security",
                    "--allow-insecure-localhost",
                    "--ignore-certificate-errors"
                ]
            }
        }
    start = time.time()
    try:
        resp = await client.post(url, json=payload, timeout=60.0)
        elapsed = (time.time() - start) * 1000
        return {"success": resp.status_code == 200, "latency_ms": elapsed, "viewport": viewport}
    except Exception as e:
        return {"success": False, "error": str(e), "viewport": viewport}
 def start_container(client, image, name, port):
    """Start container."""
    try:
        old = client.containers.get(name)
        print(f"🧹 Stopping existing container...")
        old.stop()
        old.remove()
    except docker.errors.NotFound:
        pass
    print(f"🚀 Starting container...")
    container = client.containers.run(
        image, name=name, ports={f"{port}/tcp": port},
        detach=True, shm_size="1g", mem_limit="4g",
    )
    print(f"⏳ Waiting for health...")
    for _ in range(30):
        time.sleep(1)
        container.reload()
        if container.status == "running":
            try:
                import requests
                if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200:
                    print(f"✅ Container healthy!")
                    return container
            except:
                pass
    raise TimeoutError("Container failed to start")
 async def main():
    print("="*60)
    print("TEST 5: Pool Stress - Mixed Configs")
    print("="*60)
    client = docker.from_env()
    container = None
    monitor_thread = None
    try:
        container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
        print(f"\n⏳ Waiting for permanent browser init (3s)...")
        await asyncio.sleep(3)
        # Start monitoring
        stop_monitoring.clear()
        stats_history.clear()
        monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
        monitor_thread.start()
        await asyncio.sleep(1)
        baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
        print(f"📏 Baseline: {baseline_mem:.1f} MB\n")
        url = f"http://localhost:{PORT}/crawl"
        print(f"Testing {len(VIEWPORT_CONFIGS)} different configs:")
        for i, vp in enumerate(VIEWPORT_CONFIGS):
            vp_str = "Default" if vp is None else f"{vp['width']}x{vp['height']}"
            print(f"  {i+1}. {vp_str}")
        print()
        # Run requests: repeat each config REQUESTS_PER_CONFIG times
        all_results = []
        config_sequence = []
        for _ in range(REQUESTS_PER_CONFIG):
            for viewport in VIEWPORT_CONFIGS:
                config_sequence.append(viewport)
        # Shuffle to mix configs
        random.shuffle(config_sequence)
        print(f"🔄 Running {len(config_sequence)} requests with mixed configs...")
        async with httpx.AsyncClient() as http_client:
            for i, viewport in enumerate(config_sequence):
                result = await crawl_with_viewport(http_client, url, viewport)
                all_results.append(result)
                if (i + 1) % 5 == 0:
                    vp_str = "default" if result['viewport'] is None else f"{result['viewport']['width']}x{result['viewport']['height']}"
                    status = "✓" if result.get('success') else "✗"
                    lat = f"{result.get('latency_ms', 0):.0f}ms" if 'latency_ms' in result else "error"
                    print(f"  [{i+1}/{len(config_sequence)}] {status} {vp_str} - {lat}")
        # Stop monitoring
        await asyncio.sleep(2)
        stop_monitoring.set()
        if monitor_thread:
            monitor_thread.join(timeout=2)
        # Analyze results
        pool_stats = analyze_pool_logs(container)
        successes = sum(1 for r in all_results if r.get("success"))
        success_rate = (successes / len(all_results)) * 100
        latencies = [r["latency_ms"] for r in all_results if "latency_ms" in r]
        avg_lat = sum(latencies) / len(latencies) if latencies else 0
        memory_samples = [s['memory_mb'] for s in stats_history]
        peak_mem = max(memory_samples) if memory_samples else 0
        final_mem = memory_samples[-1] if memory_samples else 0
        print(f"\n{'='*60}")
        print(f"RESULTS:")
        print(f"{'='*60}")
        print(f"  Requests:     {len(all_results)}")
        print(f"  Success Rate: {success_rate:.1f}% ({successes}/{len(all_results)})")
        print(f"  Avg Latency:  {avg_lat:.0f}ms")
        print(f"\n  Pool Statistics:")
        print(f"    🔥 Permanent: {pool_stats['permanent']}")
        print(f"    ♨️  Hot:       {pool_stats['hot']}")
        print(f"    ❄️  Cold:      {pool_stats['cold']}")
        print(f"    🆕 New:       {pool_stats['new']}")
        print(f"    ⬆️  Promotions: {pool_stats['promotions']}")
        print(f"    📊 Reuse:     {(pool_stats['total'] / len(all_results) * 100):.1f}%")
        print(f"\n  Memory:")
        print(f"    Baseline: {baseline_mem:.1f} MB")
        print(f"    Peak:     {peak_mem:.1f} MB")
        print(f"    Final:    {final_mem:.1f} MB")
        print(f"    Delta:    {final_mem - baseline_mem:+.1f} MB")
        print(f"{'='*60}")
        # Pass/Fail
        passed = True
        if success_rate < 99:
            print(f"❌ FAIL: Success rate {success_rate:.1f}% < 99%")
            passed = False
        # Should see promotions since we repeat each config 5 times
        if pool_stats['promotions'] < (len(VIEWPORT_CONFIGS) - 1):  # -1 for default
            print(f"⚠️  WARNING: Only {pool_stats['promotions']} promotions (expected ~{len(VIEWPORT_CONFIGS)-1})")
        # Should have created some browsers for different configs
        if pool_stats['new'] == 0:
            print(f"⚠️  NOTE: No new browsers created (all used default?)")
        if pool_stats['permanent'] == len(all_results):
            print(f"⚠️  NOTE: All requests used permanent browser (configs not varying enough?)")
        if final_mem - baseline_mem > 500:
            print(f"⚠️  WARNING: Memory grew {final_mem - baseline_mem:.1f} MB")
        if passed:
            print(f"✅ TEST PASSED")
            return 0
        else:
            return 1
    except Exception as e:
        print(f"\n❌ TEST ERROR: {e}")
        import traceback
        traceback.print_exc()
        return 1
    finally:
        stop_monitoring.set()
        if container:
            print(f"🛑 Stopping container...")
            container.stop()
            container.remove()
 if __name__ == "__main__":
    exit_code = asyncio.run(main())
    exit(exit_code)
--- a/deploy/docker/tests/codebase_test/test_6_multi_endpoint.py
+++ b/deploy/docker/tests/codebase_test/test_6_multi_endpoint.py
@@ -1,234 +0,0 @@
 #!/usr/bin/env python3
 """
 Test 6: Multi-Endpoint Testing
 - Tests multiple endpoints together: /html, /screenshot, /pdf, /crawl
 - Validates each endpoint works correctly
 - Monitors success rates per endpoint
 """
 import asyncio
 import time
 import docker
 import httpx
 from threading import Thread, Event
 # Config
 IMAGE = "crawl4ai-local:latest"
 CONTAINER_NAME = "crawl4ai-test"
 PORT = 11235
 REQUESTS_PER_ENDPOINT = 10
 # Stats
 stats_history = []
 stop_monitoring = Event()
 def monitor_stats(container):
    """Background stats collector."""
    for stat in container.stats(decode=True, stream=True):
        if stop_monitoring.is_set():
            break
        try:
            mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
            stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage})
        except:
            pass
        time.sleep(0.5)
 async def test_html(client, base_url, count):
    """Test /html endpoint."""
    url = f"{base_url}/html"
    results = []
    for _ in range(count):
        start = time.time()
        try:
            resp = await client.post(url, json={"url": "https://httpbin.org/html"}, timeout=30.0)
            elapsed = (time.time() - start) * 1000
            results.append({"success": resp.status_code == 200, "latency_ms": elapsed})
        except Exception as e:
            results.append({"success": False, "error": str(e)})
    return results
 async def test_screenshot(client, base_url, count):
    """Test /screenshot endpoint."""
    url = f"{base_url}/screenshot"
    results = []
    for _ in range(count):
        start = time.time()
        try:
            resp = await client.post(url, json={"url": "https://httpbin.org/html"}, timeout=30.0)
            elapsed = (time.time() - start) * 1000
            results.append({"success": resp.status_code == 200, "latency_ms": elapsed})
        except Exception as e:
            results.append({"success": False, "error": str(e)})
    return results
 async def test_pdf(client, base_url, count):
    """Test /pdf endpoint."""
    url = f"{base_url}/pdf"
    results = []
    for _ in range(count):
        start = time.time()
        try:
            resp = await client.post(url, json={"url": "https://httpbin.org/html"}, timeout=30.0)
            elapsed = (time.time() - start) * 1000
            results.append({"success": resp.status_code == 200, "latency_ms": elapsed})
        except Exception as e:
            results.append({"success": False, "error": str(e)})
    return results
 async def test_crawl(client, base_url, count):
    """Test /crawl endpoint."""
    url = f"{base_url}/crawl"
    results = []
    payload = {
        "urls": ["https://httpbin.org/html"],
        "browser_config": {},
        "crawler_config": {}
    }
    for _ in range(count):
        start = time.time()
        try:
            resp = await client.post(url, json=payload, timeout=30.0)
            elapsed = (time.time() - start) * 1000
            results.append({"success": resp.status_code == 200, "latency_ms": elapsed})
        except Exception as e:
            results.append({"success": False, "error": str(e)})
    return results
 def start_container(client, image, name, port):
    """Start container."""
    try:
        old = client.containers.get(name)
        print(f"🧹 Stopping existing container...")
        old.stop()
        old.remove()
    except docker.errors.NotFound:
        pass
    print(f"🚀 Starting container...")
    container = client.containers.run(
        image, name=name, ports={f"{port}/tcp": port},
        detach=True, shm_size="1g", mem_limit="4g",
    )
    print(f"⏳ Waiting for health...")
    for _ in range(30):
        time.sleep(1)
        container.reload()
        if container.status == "running":
            try:
                import requests
                if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200:
                    print(f"✅ Container healthy!")
                    return container
            except:
                pass
    raise TimeoutError("Container failed to start")
 async def main():
    print("="*60)
    print("TEST 6: Multi-Endpoint Testing")
    print("="*60)
    client = docker.from_env()
    container = None
    monitor_thread = None
    try:
        container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
        print(f"\n⏳ Waiting for permanent browser init (3s)...")
        await asyncio.sleep(3)
        # Start monitoring
        stop_monitoring.clear()
        stats_history.clear()
        monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
        monitor_thread.start()
        await asyncio.sleep(1)
        baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
        print(f"📏 Baseline: {baseline_mem:.1f} MB\n")
        base_url = f"http://localhost:{PORT}"
        # Test each endpoint
        endpoints = {
            "/html": test_html,
            "/screenshot": test_screenshot,
            "/pdf": test_pdf,
            "/crawl": test_crawl,
        }
        all_endpoint_stats = {}
        async with httpx.AsyncClient() as http_client:
            for endpoint_name, test_func in endpoints.items():
                print(f"🔄 Testing {endpoint_name} ({REQUESTS_PER_ENDPOINT} requests)...")
                results = await test_func(http_client, base_url, REQUESTS_PER_ENDPOINT)
                successes = sum(1 for r in results if r.get("success"))
                success_rate = (successes / len(results)) * 100
                latencies = [r["latency_ms"] for r in results if "latency_ms" in r]
                avg_lat = sum(latencies) / len(latencies) if latencies else 0
                all_endpoint_stats[endpoint_name] = {
                    'success_rate': success_rate,
                    'avg_latency': avg_lat,
                    'total': len(results),
                    'successes': successes
                }
                print(f"  ✓ Success: {success_rate:.1f}% ({successes}/{len(results)}), Avg: {avg_lat:.0f}ms")
        # Stop monitoring
        await asyncio.sleep(1)
        stop_monitoring.set()
        if monitor_thread:
            monitor_thread.join(timeout=2)
        # Final stats
        memory_samples = [s['memory_mb'] for s in stats_history]
        peak_mem = max(memory_samples) if memory_samples else 0
        final_mem = memory_samples[-1] if memory_samples else 0
        print(f"\n{'='*60}")
        print(f"RESULTS:")
        print(f"{'='*60}")
        for endpoint, stats in all_endpoint_stats.items():
            print(f"  {endpoint:12} Success: {stats['success_rate']:5.1f}%  Avg: {stats['avg_latency']:6.0f}ms")
        print(f"\n  Memory:")
        print(f"    Baseline: {baseline_mem:.1f} MB")
        print(f"    Peak:     {peak_mem:.1f} MB")
        print(f"    Final:    {final_mem:.1f} MB")
        print(f"    Delta:    {final_mem - baseline_mem:+.1f} MB")
        print(f"{'='*60}")
        # Pass/Fail
        passed = True
        for endpoint, stats in all_endpoint_stats.items():
            if stats['success_rate'] < 100:
                print(f"❌ FAIL: {endpoint} success rate {stats['success_rate']:.1f}% < 100%")
                passed = False
        if passed:
            print(f"✅ TEST PASSED")
            return 0
        else:
            return 1
    except Exception as e:
        print(f"\n❌ TEST ERROR: {e}")
        import traceback
        traceback.print_exc()
        return 1
    finally:
        stop_monitoring.set()
        if container:
            print(f"🛑 Stopping container...")
            container.stop()
            container.remove()
 if __name__ == "__main__":
    exit_code = asyncio.run(main())
    exit(exit_code)
--- a/deploy/docker/tests/codebase_test/test_7_cleanup.py
+++ b/deploy/docker/tests/codebase_test/test_7_cleanup.py
@@ -1,199 +0,0 @@
 #!/usr/bin/env python3
 """
 Test 7: Cleanup Verification (Janitor)
 - Creates load spike then goes idle
 - Verifies memory returns to near baseline
 - Tests janitor cleanup of idle browsers
 - Monitors memory recovery time
 """
 import asyncio
 import time
 import docker
 import httpx
 from threading import Thread, Event
 # Config
 IMAGE = "crawl4ai-local:latest"
 CONTAINER_NAME = "crawl4ai-test"
 PORT = 11235
 SPIKE_REQUESTS = 20  # Create some browsers
 IDLE_TIME = 90  # Wait 90s for janitor (runs every 60s)
 # Stats
 stats_history = []
 stop_monitoring = Event()
 def monitor_stats(container):
    """Background stats collector."""
    for stat in container.stats(decode=True, stream=True):
        if stop_monitoring.is_set():
            break
        try:
            mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
            stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage})
        except:
            pass
        time.sleep(1)  # Sample every 1s for this test
 def start_container(client, image, name, port):
    """Start container."""
    try:
        old = client.containers.get(name)
        print(f"🧹 Stopping existing container...")
        old.stop()
        old.remove()
    except docker.errors.NotFound:
        pass
    print(f"🚀 Starting container...")
    container = client.containers.run(
        image, name=name, ports={f"{port}/tcp": port},
        detach=True, shm_size="1g", mem_limit="4g",
    )
    print(f"⏳ Waiting for health...")
    for _ in range(30):
        time.sleep(1)
        container.reload()
        if container.status == "running":
            try:
                import requests
                if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200:
                    print(f"✅ Container healthy!")
                    return container
            except:
                pass
    raise TimeoutError("Container failed to start")
 async def main():
    print("="*60)
    print("TEST 7: Cleanup Verification (Janitor)")
    print("="*60)
    client = docker.from_env()
    container = None
    monitor_thread = None
    try:
        container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
        print(f"\n⏳ Waiting for permanent browser init (3s)...")
        await asyncio.sleep(3)
        # Start monitoring
        stop_monitoring.clear()
        stats_history.clear()
        monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
        monitor_thread.start()
        await asyncio.sleep(2)
        baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
        print(f"📏 Baseline: {baseline_mem:.1f} MB\n")
        # Create load spike with different configs to populate pool
        print(f"🔥 Creating load spike ({SPIKE_REQUESTS} requests with varied configs)...")
        url = f"http://localhost:{PORT}/crawl"
        viewports = [
            {"width": 1920, "height": 1080},
            {"width": 1024, "height": 768},
            {"width": 375, "height": 667},
        ]
        async with httpx.AsyncClient(timeout=60.0) as http_client:
            tasks = []
            for i in range(SPIKE_REQUESTS):
                vp = viewports[i % len(viewports)]
                payload = {
                    "urls": ["https://httpbin.org/html"],
                    "browser_config": {
                        "type": "BrowserConfig",
                        "params": {
                            "viewport": {"type": "dict", "value": vp},
                            "headless": True,
                            "text_mode": True,
                            "extra_args": [
                                "--no-sandbox", "--disable-dev-shm-usage",
                                "--disable-gpu", "--disable-software-rasterizer",
                                "--disable-web-security", "--allow-insecure-localhost",
                                "--ignore-certificate-errors"
                            ]
                        }
                    },
                    "crawler_config": {}
                }
                tasks.append(http_client.post(url, json=payload))
            results = await asyncio.gather(*tasks, return_exceptions=True)
            successes = sum(1 for r in results if hasattr(r, 'status_code') and r.status_code == 200)
            print(f"  ✓ Spike completed: {successes}/{len(results)} successful")
        # Measure peak
        await asyncio.sleep(2)
        peak_mem = max([s['memory_mb'] for s in stats_history]) if stats_history else baseline_mem
        print(f"  📊 Peak memory: {peak_mem:.1f} MB (+{peak_mem - baseline_mem:.1f} MB)")
        # Now go idle and wait for janitor
        print(f"\n⏸️  Going idle for {IDLE_TIME}s (janitor cleanup)...")
        print(f"  (Janitor runs every 60s, checking for idle browsers)")
        for elapsed in range(0, IDLE_TIME, 10):
            await asyncio.sleep(10)
            current_mem = stats_history[-1]['memory_mb'] if stats_history else 0
            print(f"  [{elapsed+10:3d}s] Memory: {current_mem:.1f} MB")
        # Stop monitoring
        stop_monitoring.set()
        if monitor_thread:
            monitor_thread.join(timeout=2)
        # Analyze memory recovery
        final_mem = stats_history[-1]['memory_mb'] if stats_history else 0
        recovery_mb = peak_mem - final_mem
        recovery_pct = (recovery_mb / (peak_mem - baseline_mem) * 100) if (peak_mem - baseline_mem) > 0 else 0
        print(f"\n{'='*60}")
        print(f"RESULTS:")
        print(f"{'='*60}")
        print(f"  Memory Journey:")
        print(f"    Baseline:  {baseline_mem:.1f} MB")
        print(f"    Peak:      {peak_mem:.1f} MB  (+{peak_mem - baseline_mem:.1f} MB)")
        print(f"    Final:     {final_mem:.1f} MB  (+{final_mem - baseline_mem:.1f} MB)")
        print(f"    Recovered: {recovery_mb:.1f} MB  ({recovery_pct:.1f}%)")
        print(f"{'='*60}")
        # Pass/Fail
        passed = True
        # Should have created some memory pressure
        if peak_mem - baseline_mem < 100:
            print(f"⚠️  WARNING: Peak increase only {peak_mem - baseline_mem:.1f} MB (expected more browsers)")
        # Should recover most memory (within 100MB of baseline)
        if final_mem - baseline_mem > 100:
            print(f"⚠️  WARNING: Memory didn't recover well (still +{final_mem - baseline_mem:.1f} MB above baseline)")
        else:
            print(f"✅ Good memory recovery!")
        # Baseline + 50MB tolerance
        if final_mem - baseline_mem < 50:
            print(f"✅ Excellent cleanup (within 50MB of baseline)")
        print(f"✅ TEST PASSED")
        return 0
    except Exception as e:
        print(f"\n❌ TEST ERROR: {e}")
        import traceback
        traceback.print_exc()
        return 1
    finally:
        stop_monitoring.set()
        if container:
            print(f"🛑 Stopping container...")
            container.stop()
            container.remove()
 if __name__ == "__main__":
    exit_code = asyncio.run(main())
    exit(exit_code)
--- a/deploy/docker/tests/monitor/demo_monitor_dashboard.py
+++ b/deploy/docker/tests/monitor/demo_monitor_dashboard.py
@@ -1,164 +0,0 @@
 #!/usr/bin/env python3
 """
 Monitor Dashboard Demo Script
 Generates varied activity to showcase all monitoring features for video recording.
 """
 import httpx
 import asyncio
 import time
 from datetime import datetime
 BASE_URL = "http://localhost:11235"
 async def demo_dashboard():
    print("🎬 Monitor Dashboard Demo - Starting...\n")
    print(f"📊 Dashboard: {BASE_URL}/dashboard")
    print("=" * 60)
    async with httpx.AsyncClient(timeout=60.0) as client:
        # Phase 1: Simple requests (permanent browser)
        print("\n🔷 Phase 1: Testing permanent browser pool")
        print("-" * 60)
        for i in range(5):
            print(f"  {i+1}/5 Request to /crawl (default config)...")
            try:
                r = await client.post(
                    f"{BASE_URL}/crawl",
                    json={"urls": [f"https://httpbin.org/html?req={i}"], "crawler_config": {}}
                )
                print(f"     ✅ Status: {r.status_code}, Time: {r.elapsed.total_seconds():.2f}s")
            except Exception as e:
                print(f"     ❌ Error: {e}")
            await asyncio.sleep(1)  # Small delay between requests
        # Phase 2: Create variant browsers (different configs)
        print("\n🔶 Phase 2: Testing cold→hot pool promotion")
        print("-" * 60)
        viewports = [
            {"width": 1920, "height": 1080},
            {"width": 1280, "height": 720},
            {"width": 800, "height": 600}
        ]
        for idx, viewport in enumerate(viewports):
            print(f"  Viewport {viewport['width']}x{viewport['height']}:")
            for i in range(4):  # 4 requests each to trigger promotion at 3
                try:
                    r = await client.post(
                        f"{BASE_URL}/crawl",
                        json={
                            "urls": [f"https://httpbin.org/json?v={idx}&r={i}"],
                            "browser_config": {"viewport": viewport},
                            "crawler_config": {}
                        }
                    )
                    print(f"    {i+1}/4 ✅ {r.status_code} - Should see cold→hot after 3 uses")
                except Exception as e:
                    print(f"    {i+1}/4 ❌ {e}")
                await asyncio.sleep(0.5)
        # Phase 3: Concurrent burst (stress pool)
        print("\n🔷 Phase 3: Concurrent burst (10 parallel)")
        print("-" * 60)
        tasks = []
        for i in range(10):
            tasks.append(
                client.post(
                    f"{BASE_URL}/crawl",
                    json={"urls": [f"https://httpbin.org/delay/2?burst={i}"], "crawler_config": {}}
                )
            )
        print("  Sending 10 concurrent requests...")
        start = time.time()
        results = await asyncio.gather(*tasks, return_exceptions=True)
        elapsed = time.time() - start
        successes = sum(1 for r in results if not isinstance(r, Exception) and r.status_code == 200)
        print(f"  ✅ {successes}/10 succeeded in {elapsed:.2f}s")
        # Phase 4: Multi-endpoint coverage
        print("\n🔶 Phase 4: Testing multiple endpoints")
        print("-" * 60)
        endpoints = [
            ("/md", {"url": "https://httpbin.org/html", "f": "fit", "c": "0"}),
            ("/screenshot", {"url": "https://httpbin.org/html"}),
            ("/pdf", {"url": "https://httpbin.org/html"}),
        ]
        for endpoint, payload in endpoints:
            print(f"  Testing {endpoint}...")
            try:
                if endpoint == "/md":
                    r = await client.post(f"{BASE_URL}{endpoint}", json=payload)
                else:
                    r = await client.post(f"{BASE_URL}{endpoint}", json=payload)
                print(f"    ✅ {r.status_code}")
            except Exception as e:
                print(f"    ❌ {e}")
            await asyncio.sleep(1)
        # Phase 5: Intentional error (to populate errors tab)
        print("\n🔷 Phase 5: Generating error examples")
        print("-" * 60)
        print("  Triggering invalid URL error...")
        try:
            r = await client.post(
                f"{BASE_URL}/crawl",
                json={"urls": ["invalid://bad-url"], "crawler_config": {}}
            )
            print(f"    Response: {r.status_code}")
        except Exception as e:
            print(f"    ✅ Error captured: {type(e).__name__}")
        # Phase 6: Wait for janitor activity
        print("\n🔶 Phase 6: Waiting for janitor cleanup...")
        print("-" * 60)
        print("  Idle for 40s to allow janitor to clean cold pool browsers...")
        for i in range(40, 0, -10):
            print(f"    {i}s remaining... (Check dashboard for cleanup events)")
            await asyncio.sleep(10)
        # Phase 7: Final stats check
        print("\n🔷 Phase 7: Final dashboard state")
        print("-" * 60)
        r = await client.get(f"{BASE_URL}/monitor/health")
        health = r.json()
        print(f"  Memory: {health['container']['memory_percent']:.1f}%")
        print(f"  Browsers: Perm={health['pool']['permanent']['active']}, "
              f"Hot={health['pool']['hot']['count']}, Cold={health['pool']['cold']['count']}")
        r = await client.get(f"{BASE_URL}/monitor/endpoints/stats")
        stats = r.json()
        print(f"\n  Endpoint Stats:")
        for endpoint, data in stats.items():
            print(f"    {endpoint}: {data['count']} req, "
                  f"{data['avg_latency_ms']:.0f}ms avg, "
                  f"{data['success_rate_percent']:.1f}% success")
        r = await client.get(f"{BASE_URL}/monitor/browsers")
        browsers = r.json()
        print(f"\n  Pool Efficiency:")
        print(f"    Total browsers: {browsers['summary']['total_count']}")
        print(f"    Memory usage: {browsers['summary']['total_memory_mb']} MB")
        print(f"    Reuse rate: {browsers['summary']['reuse_rate_percent']:.1f}%")
    print("\n" + "=" * 60)
    print("✅ Demo complete! Dashboard is now populated with rich data.")
    print(f"\n📹 Recording tip: Refresh {BASE_URL}/dashboard")
    print("   You should see:")
    print("   • Active & completed requests")
    print("   • Browser pool (permanent + hot/cold)")
    print("   • Janitor cleanup events")
    print("   • Endpoint analytics")
    print("   • Memory timeline")
 if __name__ == "__main__":
    try:
        asyncio.run(demo_dashboard())
    except KeyboardInterrupt:
        print("\n\n⚠️  Demo interrupted by user")
    except Exception as e:
        print(f"\n\n❌ Demo failed: {e}")
--- a/deploy/docker/tests/monitor/test_monitor_demo.py
+++ b/deploy/docker/tests/monitor/test_monitor_demo.py
@@ -1,57 +0,0 @@
 #!/usr/bin/env python3
 """Quick test to generate monitor dashboard activity"""
 import httpx
 import asyncio
 async def test_dashboard():
    async with httpx.AsyncClient(timeout=30.0) as client:
        print("📊 Generating dashboard activity...")
        # Test 1: Simple crawl
        print("\n1️⃣ Running simple crawl...")
        r1 = await client.post(
            "http://localhost:11235/crawl",
            json={"urls": ["https://httpbin.org/html"], "crawler_config": {}}
        )
        print(f"   Status: {r1.status_code}")
        # Test 2: Multiple URLs
        print("\n2️⃣ Running multi-URL crawl...")
        r2 = await client.post(
            "http://localhost:11235/crawl",
            json={
                "urls": [
                    "https://httpbin.org/html",
                    "https://httpbin.org/json"
                ],
                "crawler_config": {}
            }
        )
        print(f"   Status: {r2.status_code}")
        # Test 3: Check monitor health
        print("\n3️⃣ Checking monitor health...")
        r3 = await client.get("http://localhost:11235/monitor/health")
        health = r3.json()
        print(f"   Memory: {health['container']['memory_percent']}%")
        print(f"   Browsers: {health['pool']['permanent']['active']}")
        # Test 4: Check requests
        print("\n4️⃣ Checking request log...")
        r4 = await client.get("http://localhost:11235/monitor/requests")
        reqs = r4.json()
        print(f"   Active: {len(reqs['active'])}")
        print(f"   Completed: {len(reqs['completed'])}")
        # Test 5: Check endpoint stats
        print("\n5️⃣ Checking endpoint stats...")
        r5 = await client.get("http://localhost:11235/monitor/endpoints/stats")
        stats = r5.json()
        for endpoint, data in stats.items():
            print(f"   {endpoint}: {data['count']} requests, {data['avg_latency_ms']}ms avg")
        print("\n✅ Dashboard should now show activity!")
        print(f"\n🌐 Open: http://localhost:11235/dashboard")
 if __name__ == "__main__":
    asyncio.run(test_dashboard())
--- a/deploy/docker/tests/requirements.txt
+++ b/deploy/docker/tests/requirements.txt
@@ -1,2 +0,0 @@
 httpx>=0.25.0
 docker>=7.0.0
--- a/deploy/docker/utils.py
+++ b/deploy/docker/utils.py
@@ -178,76 +178,4 @@ def verify_email_domain(email: str) -> bool:
        records = dns.resolver.resolve(domain, 'MX')
        return True if records else False
    except Exception as e:
-        return False
+        return False
 def get_container_memory_percent() -> float:
    """Get actual container memory usage vs limit (cgroup v1/v2 aware)."""
    try:
        # Try cgroup v2 first
        usage_path = Path("/sys/fs/cgroup/memory.current")
        limit_path = Path("/sys/fs/cgroup/memory.max")
        if not usage_path.exists():
            # Fall back to cgroup v1
            usage_path = Path("/sys/fs/cgroup/memory/memory.usage_in_bytes")
            limit_path = Path("/sys/fs/cgroup/memory/memory.limit_in_bytes")
        usage = int(usage_path.read_text())
        limit = int(limit_path.read_text())
        # Handle unlimited (v2: "max", v1: > 1e18)
        if limit > 1e18:
            import psutil
            limit = psutil.virtual_memory().total
        return (usage / limit) * 100
    except:
        # Non-container or unsupported: fallback to host
        import psutil
        return psutil.virtual_memory().percent
 def get_container_id() -> str:
    """Get current container ID (hostname in Docker)."""
    import socket
    return socket.gethostname()
 def detect_deployment_mode() -> tuple[str, list[dict]]:
    """Detect if running in single/swarm/compose mode and get container list.
    Returns:
        (mode, containers) where mode is "single"|"swarm"|"compose"
        containers is list of {id, hostname, healthy}
    """
    import socket
    my_hostname = socket.gethostname()
    # Check if we're behind nginx (Compose mode indicator)
    # In Compose, service name resolves to multiple IPs
    try:
        import socket as sock
        # Try to resolve "crawl4ai" service name (Compose service)
        try:
            addrs = sock.getaddrinfo("crawl4ai", None)
            unique_ips = set(addr[4][0] for addr in addrs)
            if len(unique_ips) > 1:
                # Multiple IPs = Compose with replicas
                containers = [
                    {"id": f"container-{i+1}", "hostname": f"crawl4ai-{i+1}", "healthy": True}
                    for i in range(len(unique_ips))
                ]
                return "compose", containers
        except:
            pass
        # Check for Swarm mode (TODO: needs swarm-specific detection)
        # For now, if hostname pattern matches swarm, detect it
        if "." in my_hostname and len(my_hostname.split(".")) > 2:
            # Swarm hostname format: service.slot.task_id
            return "swarm", [{"id": my_hostname, "hostname": my_hostname, "healthy": True}]
    except:
        pass
    # Default: single container
    return "single", [{"id": my_hostname, "hostname": my_hostname, "healthy": True}]
--- a/deploy/installer/QUICKSTART.md
+++ b/deploy/installer/QUICKSTART.md
@@ -1,147 +0,0 @@
 # Crawl4AI cnode - Quick Start Cheat Sheet
 Fast reference for getting started with cnode.
 ---
 ## 📥 Install
 ```bash
 # Install cnode
 curl -sSL https://raw.githubusercontent.com/unclecode/crawl4ai/main/deploy/installer/install-cnode.sh | bash
 ```
 ---
 ## 🚀 Launch Cluster
 ```bash
 # Single server (development)
 cnode start
 # Production cluster with 5 replicas
 cnode start --replicas 5
 # Custom port
 cnode start --replicas 3 --port 8080
 ```
 ---
 ## 📊 Check Status
 ```bash
 # View server status
 cnode status
 # View logs
 cnode logs -f
 ```
 ---
 ## ⚙️ Scale Cluster
 ```bash
 # Scale to 10 replicas (live, no downtime)
 cnode scale 10
 # Scale down to 2
 cnode scale 2
 ```
 ---
 ## 🔄 Restart/Stop
 ```bash
 # Restart server
 cnode restart
 # Stop server
 cnode stop
 ```
 ---
 ## 🌐 Test the API
 ```bash
 # Simple test - crawl example.com
 curl -X POST http://localhost:11235/crawl \
  -H "Content-Type: application/json" \
  -d '{
    "urls": ["https://example.com"],
    "priority": 10
  }'
 # Pretty print with jq
 curl -X POST http://localhost:11235/crawl \
  -H "Content-Type: application/json" \
  -d '{
    "urls": ["https://example.com"],
    "priority": 10
  }' | jq '.result.markdown' -r
 # Health check
 curl http://localhost:11235/health
 ```
 ---
 ## 📱 Monitor Dashboard
 ```bash
 # Open in browser
 open http://localhost:11235/monitor
 # Or playground
 open http://localhost:11235/playground
 ```
 ---
 ## 🐍 Python Example
 ```python
 import requests
 response = requests.post(
    "http://localhost:11235/crawl",
    json={
        "urls": ["https://example.com"],
        "priority": 10
    }
 )
 result = response.json()
 print(result['result']['markdown'])
 ```
 ---
 ## 🎯 Common Commands
 | Command | Description |
 |---------|-------------|
 | `cnode start` | Start server |
 | `cnode start -r 5` | Start with 5 replicas |
 | `cnode status` | Check status |
 | `cnode scale 10` | Scale to 10 replicas |
 | `cnode logs -f` | Follow logs |
 | `cnode restart` | Restart server |
 | `cnode stop` | Stop server |
 | `cnode --help` | Show all commands |
 ---
 ## 📚 Full Documentation
 - **User Guide:** `deploy/installer/USER_GUIDE.md`
 - **Developer Docs:** `deploy/installer/README.md`
 - **Docker Guide:** `deploy/docker/README.md`
 - **Agent Context:** `deploy/docker/AGENT.md`
 ---
 **That's it!** You're ready to crawl at scale 🚀
--- a/deploy/installer/README.md
+++ b/deploy/installer/README.md
@@ -1,345 +0,0 @@
 # Crawl4AI Node Manager (cnode) - Installation & Distribution
 This directory contains the standalone `cnode` package and installation scripts for managing Crawl4AI Docker server instances.
 ## Overview
 `cnode` is a fast, lightweight CLI tool for managing Crawl4AI Docker servers. It provides:
 - One-command deployment with automatic scaling
 - Single container for development (N=1)
 - Docker Swarm for production with built-in load balancing (N>1)
 - Docker Compose + Nginx as fallback (N>1)
 ## Directory Structure
 ```
 deploy/installer/
 ├── README.md                 # This file
 ├── cnode_pkg/                # Standalone Python package
 │   ├── __init__.py           # Package marker
 │   ├── cli.py                # CLI interface (commands)
 │   ├── server_manager.py     # Docker orchestration logic
 │   └── requirements.txt      # Python dependencies
 ├── install-cnode.sh          # Local installation script
 ├── install-cnode.sh                 # Remote installation script (for users)
 └── releases/                 # Release artifacts for distribution
 ```
 ## Installation
 ### For Users (Remote Installation)
 Users can install `cnode` directly from the web:
 ```bash
 # Install from GitHub/website
 curl -sSL https://crawl4ai.com/install-cnode.sh | bash
 # Or with wget
 wget -qO- https://crawl4ai.com/install-cnode.sh | bash
 ```
 ### For Local Testing
 Test the installation locally:
 ```bash
 cd deploy/installer
 ./install-cnode.sh
 ```
 ## Package Contents
 ### `cnode_pkg/` - Python Package
 This is a self-contained Python package with:
 - **`cli.py`**: Click-based CLI with all commands (start, stop, status, scale, logs, cleanup, restart)
 - **`server_manager.py`**: Core Docker orchestration logic
 - **`requirements.txt`**: Dependencies (click, rich, anyio, pyyaml)
 - **`__init__.py`**: Package initialization
 ### Installation Script
 **`install-cnode.sh`** does the following:
 1. Checks for Python 3.8+ and pip
 2. Checks for Docker (warns if not found)
 3. Installs Python dependencies
 4. Copies `cnode_pkg/` to `/usr/local/lib/cnode/`
 5. Creates wrapper script at `/usr/local/bin/cnode`
 6. Verifies installation
 ### Wrapper Script
 Created at `/usr/local/bin/cnode`:
 ```bash
 #!/usr/bin/env bash
 set -e
 # Find Python
 if command -v python3 &> /dev/null; then
    PYTHON_CMD="python3"
 elif command -v python &> /dev/null; then
    PYTHON_CMD="python"
 else
    echo "Error: Python 3.8+ required" >&2
    exit 1
 fi
 # Run cnode
 export PYTHONPATH="/usr/local/lib/cnode:$PYTHONPATH"
 exec $PYTHON_CMD -m cnode_pkg.cli "$@"
 ```
 ## Performance
 **Blazing Fast Startup:**
 - **~0.1 seconds** to launch
 - 49x faster than compiled binary alternatives
 - Minimal overhead, maximum responsiveness
 ## Requirements
 ### User Requirements
 - Python 3.8 or higher
 - pip (Python package manager)
 - Docker (for running servers)
 ### Dependencies (Auto-installed)
 - click >= 8.0.0 (CLI framework)
 - rich >= 13.0.0 (Terminal formatting)
 - anyio >= 3.0.0 (Async I/O)
 - pyyaml >= 6.0.0 (YAML parsing)
 ## Usage
 After installation:
 ```bash
 # Quick start
 cnode start                    # Single container on port 11235
 cnode start --replicas 5       # 5-replica cluster
 cnode status                   # Check server status
 cnode logs -f                  # Follow logs
 cnode scale 10                 # Scale to 10 replicas
 cnode stop                     # Stop server
 # Get help
 cnode --help
 cnode start --help
 ```
 ## Development Workflow
 ### Making Changes
 1. **Edit source code** in `deploy/docker/`:
   ```bash
   vim deploy/docker/cnode_cli.py
   vim deploy/docker/server_manager.py
   ```
 2. **Update package** by copying to installer:
   ```bash
   # Copy CLI
   cp deploy/docker/cnode_cli.py deploy/installer/cnode_pkg/cli.py
   # Fix imports (deploy.docker → cnode_pkg)
   sed -i 's/from deploy\.docker\./from cnode_pkg./g' deploy/installer/cnode_pkg/cli.py
   # Copy server manager
   cp deploy/docker/server_manager.py deploy/installer/cnode_pkg/server_manager.py
   ```
 3. **Test locally**:
   ```bash
   cd deploy/installer
   ./install-cnode.sh
   cnode --help
   ```
 4. **Commit both**:
   ```bash
   git add deploy/docker/cnode_cli.py
   git add deploy/installer/cnode_pkg/cli.py
   git commit -m "Update cnode: [description]"
   ```
 ### Creating a Release
 1. **Tag the release**:
   ```bash
   git tag -a v1.0.0 -m "Release v1.0.0"
   git push origin v1.0.0
   ```
 2. **Package for distribution**:
   ```bash
   cd deploy/installer
   tar -czf releases/cnode-v1.0.0.tar.gz cnode_pkg/ install-cnode.sh
   ```
 3. **Create GitHub release**:
   ```bash
   gh release create v1.0.0 \
     releases/cnode-v1.0.0.tar.gz \
     --title "cnode v1.0.0" \
     --notes "Release notes here"
   ```
 4. **Update deployment script** (if needed):
   - Update `install-cnode.sh` with new version/URL
   - Upload to hosting (e.g., `https://crawl4ai.com/install-cnode.sh`)
 ## Deployment
 ### Remote Installation Script
 The `install-cnode.sh` script is meant to be hosted at a public URL for user installation:
 ```bash
 # Upload to your server
 scp install-cnode.sh user@crawl4ai.com:/var/www/html/install-cnode.sh
 # Or use GitHub raw URL
 https://raw.githubusercontent.com/unclecode/crawl4ai/main/deploy/installer/install-cnode.sh
 ```
 Users can then install with:
 ```bash
 curl -sSL https://crawl4ai.com/install-cnode.sh | bash
 ```
 ## Backward Compatibility
 The main Crawl4AI CLI (`crwl`) includes a redirect for backward compatibility:
 ```bash
 # These work identically:
 crwl server start --replicas 3
 cnode start --replicas 3
 # All subcommands redirect:
 crwl server status  → cnode status
 crwl server stop    → cnode stop
 crwl server scale 5 → cnode scale 5
 crwl server logs -f → cnode logs -f
 ```
 This ensures existing scripts continue working while users migrate to `cnode`.
 ## Uninstallation
 To remove cnode:
 ```bash
 # Remove command
 sudo rm /usr/local/bin/cnode
 # Remove package
 sudo rm -rf /usr/local/lib/cnode
 # (Optional) Uninstall dependencies
 pip uninstall click rich anyio pyyaml
 ```
 ## Troubleshooting
 ### Python Not Found
 ```bash
 # Install Python 3.8+
 # macOS: brew install python3
 # Ubuntu: sudo apt install python3 python3-pip
 # RHEL/CentOS: sudo yum install python3 python3-pip
 ```
 ### Permission Denied
 ```bash
 # Run installer with sudo
 sudo ./install-cnode.sh
 # Or change install location
 INSTALL_DIR=$HOME/.local/bin ./install-cnode.sh
 ```
 ### Command Not Found After Install
 ```bash
 # Add to PATH in ~/.bashrc or ~/.zshrc
 export PATH="/usr/local/bin:$PATH"
 # Reload shell
 source ~/.bashrc  # or source ~/.zshrc
 ```
 ### Dependencies Install Failed
 ```bash
 # Install manually
 pip install --user click rich anyio pyyaml
 # Or with break-system-packages (if needed)
 pip install --user --break-system-packages click rich anyio pyyaml
 ```
 ### Docker Not Running
 ```bash
 # macOS: Start Docker Desktop
 # Linux: sudo systemctl start docker
 # Check Docker
 docker --version
 docker ps
 ```
 ## Architecture
 ### Component Flow
 ```
 User runs: cnode start
         ↓
 /usr/local/bin/cnode (wrapper script)
         ↓
 Finds python3 executable
         ↓
 Sets PYTHONPATH=/usr/local/lib/cnode
         ↓
 python3 -m cnode_pkg.cli start
         ↓
 cli.py → start_cmd()
         ↓
 server_manager.py → ServerManager.start()
         ↓
 Docker orchestration (single/swarm/compose)
         ↓
 Server running!
 ```
 ### Why Python Wrapper vs Binary?
 We chose a Python wrapper over compiled binaries (PyInstaller) because:
 | Metric | Python Wrapper | PyInstaller Binary |
 |--------|---------------|-------------------|
 | Startup time | **0.1s** | 4.7s |
 | Size | ~50KB wrapper | 8.8MB |
 | Updates | Easy (just copy files) | Rebuild required |
 | Dependencies | Python 3.8+ | None |
 | Platform | Any with Python | OS-specific builds |
 Since users running Crawl4AI already have Python, the wrapper is the clear winner.
 ## Support
 For issues or questions:
 - GitHub Issues: https://github.com/unclecode/crawl4ai/issues
 - Documentation: https://docs.crawl4ai.com
 - Discord: https://discord.gg/crawl4ai
 ## Version History
 - **v1.0.0**: Initial release with Python wrapper approach
  - Fast startup (~0.1s)
  - Supports single container, Docker Swarm, and Compose modes
  - Auto-scaling and load balancing
  - Real-time monitoring and logs
--- a/deploy/installer/USER_GUIDE.md
+++ b/deploy/installer/USER_GUIDE.md
@@ -1,676 +0,0 @@
 # Crawl4AI Node Manager (cnode) - User Guide 🚀
 Self-host your own Crawl4AI server cluster with one command. Scale from development to production effortlessly.
 ## Table of Contents
 - [What is cnode?](#what-is-cnode)
 - [Quick Start](#quick-start)
 - [Installation](#installation)
 - [Basic Usage](#basic-usage)
 - [Scaling & Production](#scaling--production)
 - [Monitoring Dashboard](#monitoring-dashboard)
 - [Using the API](#using-the-api)
 - [Management Commands](#management-commands)
 - [Troubleshooting](#troubleshooting)
 - [Advanced Topics](#advanced-topics)
 ---
 ## What is cnode?
 **cnode** (Crawl4AI Node Manager) is a CLI tool that manages Crawl4AI Docker server instances with automatic scaling and load balancing.
 ### Key Features
 ✅ **One-Command Deployment** - Start a server or cluster instantly
 ✅ **Automatic Scaling** - Single container or multi-replica cluster
 ✅ **Built-in Load Balancing** - Docker Swarm or Nginx (auto-detected)
 ✅ **Real-time Monitoring** - Beautiful web dashboard
 ✅ **Zero Configuration** - Works out of the box
 ✅ **Production Ready** - Auto-scaling, health checks, rolling updates
 ### Architecture Modes
 | Replicas | Mode | Load Balancer | Use Case |
 |----------|------|---------------|----------|
 | 1 | Single Container | None | Development, testing |
 | 2+ | Docker Swarm | Built-in | Production (if Swarm available) |
 | 2+ | Docker Compose | Nginx | Production (fallback) |
 ---
 ## Quick Start
 ### 1. Install cnode
 ```bash
 # One-line installation
 curl -sSL https://crawl4ai.com/install-cnode.sh | bash
 ```
 **Requirements:**
 - Python 3.8+
 - Docker
 - Git
 ### 2. Start Your First Server
 ```bash
 # Start single development server
 cnode start
 # Or start a production cluster with 5 replicas
 cnode start --replicas 5
 ```
 That's it! Your server is running at **http://localhost:11235** 🎉
 ---
 ## Installation
 ### Method 1: Quick Install (Recommended)
 ```bash
 # From crawl4ai.com (when hosted)
 curl -sSL https://crawl4ai.com/install-cnode.sh | bash
 # Or directly from GitHub
 curl -sSL https://raw.githubusercontent.com/unclecode/crawl4ai/main/deploy/installer/install-cnode.sh | bash
 ```
 ### Method 2: Clone Repository (For Development)
 ```bash
 # Clone the repository
 git clone https://github.com/unclecode/crawl4ai.git
 cd crawl4ai/deploy/installer
 # Run installer
 ./install-cnode.sh
 ```
 ### Method 3: Custom Location
 ```bash
 # Install to custom directory (using GitHub raw URL)
 INSTALL_DIR=$HOME/.local/bin curl -sSL https://raw.githubusercontent.com/unclecode/crawl4ai/main/deploy/installer/install-cnode.sh | bash
 # Add to PATH
 export PATH="$HOME/.local/bin:$PATH"
 ```
 ### Verify Installation
 ```bash
 cnode --help
 ```
 ---
 ## Basic Usage
 ### Start Server
 ```bash
 # Development server (1 replica)
 cnode start
 # Production cluster (5 replicas with auto-scaling)
 cnode start --replicas 5
 # Custom port
 cnode start --port 8080
 # Specific Docker image
 cnode start --image unclecode/crawl4ai:0.7.0
 ```
 ### Check Status
 ```bash
 cnode status
 ```
 **Example Output:**
 ```
 ╭─────────────────── Crawl4AI Server Status ───────────────────╮
 │ Status     │ 🟢 Running                                      │
 │ Mode       │ swarm                                           │
 │ Replicas   │ 5                                               │
 │ Port       │ 11235                                           │
 │ Image      │ unclecode/crawl4ai:latest                       │
 │ Uptime     │ 2 hours 34 minutes                              │
 │ Started    │ 2025-10-21 14:30:00                            │
 ╰─────────────────────────────────────────────────────────────╯
 ✓ Server is healthy
 Access: http://localhost:11235
 ```
 ### View Logs
 ```bash
 # Show last 100 lines
 cnode logs
 # Follow logs in real-time
 cnode logs -f
 # Show last 500 lines
 cnode logs --tail 500
 ```
 ### Stop Server
 ```bash
 # Stop server (keeps data)
 cnode stop
 # Stop and remove all data
 cnode stop --remove-volumes
 ```
 ---
 ## Scaling & Production
 ### Scale Your Cluster
 ```bash
 # Scale to 10 replicas (live, no downtime)
 cnode scale 10
 # Scale down to 2 replicas
 cnode scale 2
 ```
 **Note:** Scaling is live for Swarm/Compose modes. Single container mode requires restart.
 ### Production Deployment
 ```bash
 # Start production cluster
 cnode start --replicas 5 --port 11235
 # Verify health
 curl http://localhost:11235/health
 # Monitor performance
 cnode logs -f
 ```
 ### Restart Server
 ```bash
 # Restart with same configuration
 cnode restart
 # Restart with new replica count
 cnode restart --replicas 10
 ```
 ---
 ## Monitoring Dashboard
 ### Access the Dashboard
 Once your server is running, access the real-time monitoring dashboard:
 ```bash
 # Dashboard URL
 http://localhost:11235/monitor
 ```
 ### Dashboard Features
 📊 **Real-time Metrics**
 - Requests per second
 - Active connections
 - Response times
 - Error rates
 📈 **Performance Graphs**
 - CPU usage
 - Memory consumption
 - Request latency
 - Throughput
 🔍 **System Health**
 - Container status
 - Replica health
 - Load distribution
 - Resource utilization
 ![Monitor Dashboard](https://crawl4ai.com/images/monitor-dashboard.png)
 ### API Health Endpoint
 ```bash
 # Quick health check
 curl http://localhost:11235/health
 # Response
 {
  "status": "healthy",
  "version": "1.0.0",
  "uptime": 9876,
  "containers": 5
 }
 ```
 ---
 ## Using the API
 ### Interactive Playground
 Test the API interactively:
 ```
 http://localhost:11235/playground
 ```
 ### Basic Crawl Example
 **Python:**
 ```python
 import requests
 # Simple crawl
 response = requests.post(
    "http://localhost:11235/crawl",
    json={
        "urls": ["https://example.com"],
        "browser_config": {
            "type": "BrowserConfig",
            "params": {"headless": True}
        },
        "crawler_config": {
            "type": "CrawlerRunConfig",
            "params": {"cache_mode": "bypass"}
        }
    }
 )
 result = response.json()
 print(f"Title: {result['result']['metadata']['title']}")
 print(f"Content: {result['result']['markdown'][:200]}...")
 ```
 **cURL:**
 ```bash
 curl -X POST http://localhost:11235/crawl \
  -H "Content-Type: application/json" \
  -d '{
    "urls": ["https://example.com"],
    "browser_config": {
      "type": "BrowserConfig",
      "params": {"headless": true}
    },
    "crawler_config": {
      "type": "CrawlerRunConfig",
      "params": {"cache_mode": "bypass"}
    }
  }'
 ```
 **JavaScript (Node.js):**
 ```javascript
 const axios = require('axios');
 async function crawl() {
  const response = await axios.post('http://localhost:11235/crawl', {
    urls: ['https://example.com'],
    browser_config: {
      type: 'BrowserConfig',
      params: { headless: true }
    },
    crawler_config: {
      type: 'CrawlerRunConfig',
      params: { cache_mode: 'bypass' }
    }
  });
  console.log('Title:', response.data.result.metadata.title);
  console.log('Content:', response.data.result.markdown.substring(0, 200));
 }
 crawl();
 ```
 ### Advanced Examples
 **Extract with CSS Selectors:**
 ```python
 import requests
 response = requests.post(
    "http://localhost:11235/crawl",
    json={
        "urls": ["https://news.ycombinator.com"],
        "browser_config": {
            "type": "BrowserConfig",
            "params": {"headless": True}
        },
        "crawler_config": {
            "type": "CrawlerRunConfig",
            "params": {
                "extraction_strategy": {
                    "type": "JsonCssExtractionStrategy",
                    "params": {
                        "schema": {
                            "type": "dict",
                            "value": {
                                "baseSelector": ".athing",
                                "fields": [
                                    {"name": "title", "selector": ".titleline > a", "type": "text"},
                                    {"name": "url", "selector": ".titleline > a", "type": "attribute", "attribute": "href"},
                                    {"name": "points", "selector": ".score", "type": "text"}
                                ]
                            }
                        }
                    }
                }
            }
        }
    }
 )
 articles = response.json()['result']['extracted_content']
 for article in articles:
    print(f"{article['title']} - {article['points']}")
 ```
 **Streaming Multiple URLs:**
 ```python
 import requests
 import json
 response = requests.post(
    "http://localhost:11235/crawl/stream",
    json={
        "urls": [
            "https://example.com",
            "https://httpbin.org/html",
            "https://python.org"
        ],
        "browser_config": {
            "type": "BrowserConfig",
            "params": {"headless": True}
        },
        "crawler_config": {
            "type": "CrawlerRunConfig",
            "params": {"stream": True}
        }
    },
    stream=True
 )
 for line in response.iter_lines():
    if line:
        data = json.loads(line)
        if data.get("status") == "completed":
            break
        print(f"Crawled: {data['url']} - Success: {data['success']}")
 ```
 ### Additional Endpoints
 **Screenshot:**
 ```bash
 curl -X POST http://localhost:11235/screenshot \
  -H "Content-Type: application/json" \
  -d '{"url": "https://example.com"}' \
  --output screenshot.png
 ```
 **PDF Export:**
 ```bash
 curl -X POST http://localhost:11235/pdf \
  -H "Content-Type: application/json" \
  -d '{"url": "https://example.com"}' \
  --output page.pdf
 ```
 **HTML Extraction:**
 ```bash
 curl -X POST http://localhost:11235/html \
  -H "Content-Type: application/json" \
  -d '{"url": "https://example.com"}'
 ```
 ---
 ## Management Commands
 ### All Available Commands
 ```bash
 cnode --help              # Show help
 cnode start [OPTIONS]     # Start server
 cnode stop [OPTIONS]      # Stop server
 cnode status              # Show status
 cnode scale N             # Scale to N replicas
 cnode logs [OPTIONS]      # View logs
 cnode restart [OPTIONS]   # Restart server
 cnode cleanup [--force]   # Clean up resources
 ```
 ### Command Options
 **start:**
 ```bash
 --replicas, -r N      # Number of replicas (default: 1)
 --mode MODE           # Deployment mode: auto, single, swarm, compose
 --port, -p PORT       # External port (default: 11235)
 --env-file FILE       # Environment file path
 --image IMAGE         # Docker image (default: unclecode/crawl4ai:latest)
 ```
 **stop:**
 ```bash
 --remove-volumes      # Remove persistent data (WARNING: deletes data)
 ```
 **logs:**
 ```bash
 --follow, -f          # Follow log output (like tail -f)
 --tail N              # Number of lines to show (default: 100)
 ```
 **scale:**
 ```bash
 N                     # Target replica count (minimum: 1)
 ```
 ---
 ## Troubleshooting
 ### Server Won't Start
 ```bash
 # Check Docker is running
 docker ps
 # Check port availability
 lsof -i :11235
 # Check logs for errors
 cnode logs
 ```
 ### High Memory Usage
 ```bash
 # Check current status
 cnode status
 # Restart to clear memory
 cnode restart
 # Scale down if needed
 cnode scale 2
 ```
 ### Slow Response Times
 ```bash
 # Scale up for better performance
 cnode scale 10
 # Check system resources
 docker stats
 ```
 ### Cannot Connect to API
 ```bash
 # Verify server is running
 cnode status
 # Check firewall
 sudo ufw status
 # Test locally
 curl http://localhost:11235/health
 ```
 ### Clean Slate
 ```bash
 # Complete cleanup and restart
 cnode cleanup --force
 cnode start --replicas 5
 ```
 ---
 ## Advanced Topics
 ### Environment Variables
 Create `.env` file for API keys:
 ```bash
 # .env file
 OPENAI_API_KEY=sk-your-key
 ANTHROPIC_API_KEY=your-key
 ```
 Use with cnode:
 ```bash
 cnode start --env-file .env --replicas 3
 ```
 ### Custom Docker Image
 ```bash
 # Use specific version
 cnode start --image unclecode/crawl4ai:0.7.0-r1
 # Use custom registry
 cnode start --image myregistry.com/crawl4ai:custom
 ```
 ### Production Best Practices
 1. **Use Multiple Replicas**
   ```bash
   cnode start --replicas 5
   ```
 2. **Monitor Regularly**
   ```bash
   # Set up monitoring cron
   */5 * * * * cnode status | mail -s "Crawl4AI Status" admin@example.com
   ```
 3. **Regular Log Rotation**
   ```bash
   cnode logs --tail 1000 > crawl4ai.log
   cnode restart
   ```
 4. **Resource Limits**
   - Ensure adequate RAM (2GB per replica minimum)
   - Monitor disk space for cached data
   - Use SSD for better performance
 ### Integration Examples
 **With Python App:**
 ```python
 from crawl4ai.docker_client import Crawl4aiDockerClient
 async def main():
    async with Crawl4aiDockerClient(base_url="http://localhost:11235") as client:
        results = await client.crawl(["https://example.com"])
        print(results[0].markdown)
 ```
 **With Node.js:**
 ```javascript
 const Crawl4AI = require('crawl4ai-client');
 const client = new Crawl4AI('http://localhost:11235');
 client.crawl('https://example.com')
  .then(result => console.log(result.markdown));
 ```
 **With REST API:**
 Any language with HTTP client support can use the API!
 ---
 ## Getting Help
 ### Resources
 - 📖 [Full Documentation](https://docs.crawl4ai.com)
 - 🐛 [Report Issues](https://github.com/unclecode/crawl4ai/issues)
 - 💬 [Discord Community](https://discord.gg/crawl4ai)
 - 📺 [Video Tutorials](https://youtube.com/@crawl4ai)
 ### Common Questions
 **Q: How many replicas should I use?**
 A: Start with 1 for development. Use 3-5 for production. Scale based on load.
 **Q: What's the difference between Swarm and Compose mode?**
 A: Swarm has built-in load balancing (faster). Compose uses Nginx (fallback if Swarm unavailable).
 **Q: Can I run multiple cnode instances?**
 A: Yes! Use different ports: `cnode start --port 8080`
 **Q: How do I update to the latest version?**
 A: Pull new image: `cnode stop && docker pull unclecode/crawl4ai:latest && cnode start`
 ---
 ## Summary
 You now know how to:
 - ✅ Install cnode with one command
 - ✅ Start and manage Crawl4AI servers
 - ✅ Scale from 1 to 100+ replicas
 - ✅ Monitor performance in real-time
 - ✅ Use the API from any language
 - ✅ Troubleshoot common issues
 **Ready to crawl at scale!** 🚀
 For detailed Docker configuration and advanced deployment options, see the [Docker Guide](../docker/README.md).
 ---
 **Happy Crawling!** 🕷️
 *Made with ❤️ by the Crawl4AI team*
--- a/deploy/installer/cnode_pkg/init.py
+++ b/deploy/installer/cnode_pkg/init.py
@@ -1,5 +0,0 @@
 """
 Crawl4AI Node Manager (cnode) - Docker server orchestration CLI
 """
 __version__ = "1.0.0"
--- a/deploy/installer/cnode_pkg/cli.py
+++ b/deploy/installer/cnode_pkg/cli.py
@@ -1,492 +0,0 @@
 """
 Crawl4AI Server CLI Commands
 Provides `cnode` command group for Docker orchestration.
 """
 import click
 import anyio
 from rich.console import Console
 from rich.table import Table
 from rich.panel import Panel
 from rich.prompt import Confirm
 from cnode_pkg.server_manager import ServerManager
 console = Console()
@click.group()
 def cli():
    """Manage Crawl4AI Docker server instances
    \b
    One-command deployment with automatic scaling:
      • Single container for development (N=1)
      • Docker Swarm for production with built-in load balancing (N>1)
      • Docker Compose + Nginx as fallback (N>1)
    \b
    Examples:
      cnode start                    # Single container on port 11235
      cnode start --replicas 3       # Auto-detect Swarm or Compose
      cnode start -r 5 --port 8080   # 5 replicas on custom port
      cnode status                   # Check current deployment
      cnode scale 10                 # Scale to 10 replicas
      cnode stop                     # Stop and cleanup
    """
    pass
@cli.command("start")
@click.option(
    "--replicas", "-r",
    type=int,
    default=1,
    help="Number of container replicas (default: 1)"
 )
@click.option(
    "--mode",
    type=click.Choice(["auto", "single", "swarm", "compose"]),
    default="auto",
    help="Deployment mode (default: auto-detect)"
 )
@click.option(
    "--port", "-p",
    type=int,
    default=11235,
    help="External port to expose (default: 11235)"
 )
@click.option(
    "--env-file",
    type=click.Path(exists=True),
    help="Path to environment file"
 )
@click.option(
    "--image",
    default="unclecode/crawl4ai:latest",
    help="Docker image to use (default: unclecode/crawl4ai:latest)"
 )
 def start_cmd(replicas: int, mode: str, port: int, env_file: str, image: str):
    """Start Crawl4AI server with automatic orchestration.
    Deployment modes:
    - auto: Automatically choose best mode (default)
    - single: Single container (N=1 only)
    - swarm: Docker Swarm with built-in load balancing
    - compose: Docker Compose + Nginx reverse proxy
    The server will:
    1. Check if Docker is running
    2. Validate port availability
    3. Pull image if needed
    4. Start container(s) with health checks
    5. Save state for management
    Examples:
        # Development: single container
        cnode start
        # Production: 5 replicas with Swarm
        cnode start --replicas 5
        # Custom configuration
        cnode start -r 3 --port 8080 --env-file .env.prod
    """
    manager = ServerManager()
    console.print(Panel(
        f"[cyan]Starting Crawl4AI Server[/cyan]\n\n"
        f"Replicas: [yellow]{replicas}[/yellow]\n"
        f"Mode: [yellow]{mode}[/yellow]\n"
        f"Port: [yellow]{port}[/yellow]\n"
        f"Image: [yellow]{image}[/yellow]",
        title="Server Start",
        border_style="cyan"
    ))
    with console.status("[cyan]Starting server..."):
        async def _start():
            return await manager.start(
                replicas=replicas,
                mode=mode,
                port=port,
                env_file=env_file,
                image=image
            )
        result = anyio.run(_start)
    if result["success"]:
        console.print(Panel(
            f"[green]✓ Server started successfully![/green]\n\n"
            f"Mode: [cyan]{result.get('state_data', {}).get('mode', mode)}[/cyan]\n"
            f"URL: [bold]http://localhost:{port}[/bold]\n"
            f"Health: [bold]http://localhost:{port}/health[/bold]\n"
            f"Monitor: [bold]http://localhost:{port}/monitor[/bold]",
            title="Server Running",
            border_style="green"
        ))
    else:
        error_msg = result.get("error", result.get("message", "Unknown error"))
        console.print(Panel(
            f"[red]✗ Failed to start server[/red]\n\n"
            f"{error_msg}",
            title="Error",
            border_style="red"
        ))
        if "already running" in error_msg.lower():
            console.print("\n[yellow]Hint: Use 'cnode status' to check current deployment[/yellow]")
            console.print("[yellow]      Use 'cnode stop' to stop existing server[/yellow]")
@cli.command("status")
 def status_cmd():
    """Show current server status and deployment info.
    Displays:
    - Running state (up/down)
    - Deployment mode (single/swarm/compose)
    - Number of replicas
    - Port mapping
    - Uptime
    - Image version
    Example:
        cnode status
    """
    manager = ServerManager()
    async def _status():
        return await manager.status()
    result = anyio.run(_status)
    if result["running"]:
        table = Table(title="Crawl4AI Server Status", border_style="green")
        table.add_column("Property", style="cyan")
        table.add_column("Value", style="green")
        table.add_row("Status", "🟢 Running")
        table.add_row("Mode", result["mode"])
        table.add_row("Replicas", str(result.get("replicas", 1)))
        table.add_row("Port", str(result.get("port", 11235)))
        table.add_row("Image", result.get("image", "unknown"))
        table.add_row("Uptime", result.get("uptime", "unknown"))
        table.add_row("Started", result.get("started_at", "unknown"))
        console.print(table)
        console.print(f"\n[green]✓ Server is healthy[/green]")
        console.print(f"[dim]Access: http://localhost:{result.get('port', 11235)}[/dim]")
    else:
        console.print(Panel(
            f"[yellow]No server is currently running[/yellow]\n\n"
            f"Use 'cnode start' to launch a server",
            title="Server Status",
            border_style="yellow"
        ))
@cli.command("stop")
@click.option(
    "--remove-volumes",
    is_flag=True,
    help="Remove associated volumes (WARNING: deletes data)"
 )
 def stop_cmd(remove_volumes: bool):
    """Stop running Crawl4AI server and cleanup resources.
    This will:
    1. Stop all running containers/services
    2. Remove containers
    3. Optionally remove volumes (--remove-volumes)
    4. Clean up state files
    WARNING: Use --remove-volumes with caution as it will delete
    persistent data including Redis databases and logs.
    Examples:
        # Stop server, keep volumes
        cnode stop
        # Stop and remove all data
        cnode stop --remove-volumes
    """
    manager = ServerManager()
    # Confirm if removing volumes
    if remove_volumes:
        if not Confirm.ask(
            "[red]⚠️  This will delete all server data including Redis databases. Continue?[/red]"
        ):
            console.print("[yellow]Cancelled[/yellow]")
            return
    with console.status("[cyan]Stopping server..."):
        async def _stop():
            return await manager.stop(remove_volumes=remove_volumes)
        result = anyio.run(_stop)
    if result["success"]:
        console.print(Panel(
            f"[green]✓ Server stopped successfully[/green]\n\n"
            f"{result.get('message', 'All resources cleaned up')}",
            title="Server Stopped",
            border_style="green"
        ))
    else:
        console.print(Panel(
            f"[red]✗ Error stopping server[/red]\n\n"
            f"{result.get('error', result.get('message', 'Unknown error'))}",
            title="Error",
            border_style="red"
        ))
@cli.command("scale")
@click.argument("replicas", type=int)
 def scale_cmd(replicas: int):
    """Scale server to specified number of replicas.
    Only works with Swarm or Compose modes. Single container
    mode cannot be scaled (must stop and restart with --replicas).
    Scaling is live and does not require downtime. The load
    balancer will automatically distribute traffic to new replicas.
    Examples:
        # Scale up to 10 replicas
        cnode scale 10
        # Scale down to 2 replicas
        cnode scale 2
        # Scale to 1 (minimum)
        cnode scale 1
    """
    if replicas < 1:
        console.print("[red]Error: Replicas must be at least 1[/red]")
        return
    manager = ServerManager()
    with console.status(f"[cyan]Scaling to {replicas} replicas..."):
        async def _scale():
            return await manager.scale(replicas=replicas)
        result = anyio.run(_scale)
    if result["success"]:
        console.print(Panel(
            f"[green]✓ Scaled successfully[/green]\n\n"
            f"New replica count: [bold]{replicas}[/bold]\n"
            f"Mode: [cyan]{result.get('mode')}[/cyan]",
            title="Scaling Complete",
            border_style="green"
        ))
    else:
        error_msg = result.get("error", result.get("message", "Unknown error"))
        console.print(Panel(
            f"[red]✗ Scaling failed[/red]\n\n"
            f"{error_msg}",
            title="Error",
            border_style="red"
        ))
        if "single container" in error_msg.lower():
            console.print("\n[yellow]Hint: For single container mode:[/yellow]")
            console.print("[yellow]  1. cnode stop[/yellow]")
            console.print(f"[yellow]  2. cnode start --replicas {replicas}[/yellow]")
@cli.command("logs")
@click.option(
    "--follow", "-f",
    is_flag=True,
    help="Follow log output (like tail -f)"
 )
@click.option(
    "--tail",
    type=int,
    default=100,
    help="Number of lines to show (default: 100)"
 )
 def logs_cmd(follow: bool, tail: int):
    """View server logs.
    Shows logs from running containers/services. Use --follow
    to stream logs in real-time.
    Examples:
        # Show last 100 lines
        cnode logs
        # Show last 500 lines
        cnode logs --tail 500
        # Follow logs in real-time
        cnode logs --follow
        # Combine options
        cnode logs -f --tail 50
    """
    manager = ServerManager()
    async def _logs():
        return await manager.logs(follow=follow, tail=tail)
    output = anyio.run(_logs)
    console.print(output)
@cli.command("cleanup")
@click.option(
    "--force",
    is_flag=True,
    help="Force cleanup even if state file doesn't exist"
 )
 def cleanup_cmd(force: bool):
    """Force cleanup of all Crawl4AI Docker resources.
    Stops and removes all containers, networks, and optionally volumes.
    Useful when server is stuck or state is corrupted.
    Examples:
        # Clean up everything
        cnode cleanup
        # Force cleanup (ignore state file)
        cnode cleanup --force
    """
    manager = ServerManager()
    console.print(Panel(
        f"[yellow]⚠️  Cleaning up Crawl4AI Docker resources[/yellow]\n\n"
        f"This will stop and remove:\n"
        f"- All Crawl4AI containers\n"
        f"- Nginx load balancer\n"
        f"- Redis instance\n"
        f"- Docker networks\n"
        f"- State files",
        title="Cleanup",
        border_style="yellow"
    ))
    if not force and not Confirm.ask("[yellow]Continue with cleanup?[/yellow]"):
        console.print("[yellow]Cancelled[/yellow]")
        return
    with console.status("[cyan]Cleaning up resources..."):
        async def _cleanup():
            return await manager.cleanup(force=force)
        result = anyio.run(_cleanup)
    if result["success"]:
        console.print(Panel(
            f"[green]✓ Cleanup completed successfully[/green]\n\n"
            f"Removed: {result.get('removed', 0)} containers\n"
            f"{result.get('message', 'All resources cleaned up')}",
            title="Cleanup Complete",
            border_style="green"
        ))
    else:
        console.print(Panel(
            f"[yellow]⚠️  Partial cleanup[/yellow]\n\n"
            f"{result.get('message', 'Some resources may still exist')}",
            title="Cleanup Status",
            border_style="yellow"
        ))
@cli.command("restart")
@click.option(
    "--replicas", "-r",
    type=int,
    help="New replica count (optional)"
 )
 def restart_cmd(replicas: int):
    """Restart server (stop then start with same config).
    Preserves existing configuration unless overridden with options.
    Useful for applying image updates or recovering from errors.
    Examples:
        # Restart with same configuration
        cnode restart
        # Restart and change replica count
        cnode restart --replicas 5
    """
    manager = ServerManager()
    # Get current state
    async def _get_status():
        return await manager.status()
    current = anyio.run(_get_status)
    if not current["running"]:
        console.print("[yellow]No server is running. Use 'cnode start' instead.[/yellow]")
        return
    # Extract current config
    current_replicas = current.get("replicas", 1)
    current_port = current.get("port", 11235)
    current_image = current.get("image", "unclecode/crawl4ai:latest")
    current_mode = current.get("mode", "auto")
    # Override with CLI args
    new_replicas = replicas if replicas is not None else current_replicas
    console.print(Panel(
        f"[cyan]Restarting Crawl4AI Server[/cyan]\n\n"
        f"Replicas: [yellow]{current_replicas}[/yellow] → [green]{new_replicas}[/green]\n"
        f"Port: [yellow]{current_port}[/yellow]\n"
        f"Mode: [yellow]{current_mode}[/yellow]",
        title="Server Restart",
        border_style="cyan"
    ))
    # Stop current
    with console.status("[cyan]Stopping current server..."):
        async def _stop_server():
            return await manager.stop(remove_volumes=False)
        stop_result = anyio.run(_stop_server)
    if not stop_result["success"]:
        console.print(f"[red]Failed to stop server: {stop_result.get('error')}[/red]")
        return
    # Start new
    with console.status("[cyan]Starting server..."):
        async def _start_server():
            return await manager.start(
                replicas=new_replicas,
                mode="auto",
                port=current_port,
                image=current_image
            )
        start_result = anyio.run(_start_server)
    if start_result["success"]:
        console.print(Panel(
            f"[green]✓ Server restarted successfully![/green]\n\n"
            f"URL: [bold]http://localhost:{current_port}[/bold]",
            title="Restart Complete",
            border_style="green"
        ))
    else:
        console.print(Panel(
            f"[red]✗ Failed to restart server[/red]\n\n"
            f"{start_result.get('error', 'Unknown error')}",
            title="Error",
            border_style="red"
        ))
 def main():
    """Entry point for cnode CLI"""
    cli()
 if __name__ == "__main__":
    main()
 # Test comment
--- a/deploy/installer/cnode_pkg/requirements.txt
+++ b/deploy/installer/cnode_pkg/requirements.txt
@@ -1,4 +0,0 @@
 click>=8.0.0
 rich>=13.0.0
 anyio>=3.0.0
 pyyaml>=6.0.0
--- a/deploy/installer/cnode_pkg/server_manager.py
+++ b/deploy/installer/cnode_pkg/server_manager.py
--- a/deploy/installer/install-cnode.sh
+++ b/deploy/installer/install-cnode.sh
@@ -1,171 +0,0 @@
 #!/bin/bash
 # Crawl4AI Node Manager (cnode) Remote Installation Script
 # Usage: curl -sSL https://crawl4ai.com/install-cnode.sh | bash
 # Or: wget -qO- https://crawl4ai.com/install-cnode.sh | bash
 set -e
 # Colors
 RED='\033[0;31m'
 GREEN='\033[0;32m'
 YELLOW='\033[1;33m'
 BLUE='\033[0;34m'
 NC='\033[0m'
 # Configuration
 INSTALL_DIR="${INSTALL_DIR:-/usr/local/bin}"
 LIB_DIR="${LIB_DIR:-/usr/local/lib/cnode}"
 GITHUB_REPO="unclecode/crawl4ai"
 BRANCH="${CNODE_BRANCH:-main}"
 echo -e "${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}"
 echo -e "${GREEN}║   Crawl4AI Node Manager (cnode) Installation Script         ║${NC}"
 echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}\n"
 # Check Python
 echo -e "${BLUE}Checking Python installation...${NC}"
 if command -v python3 &> /dev/null; then
    PYTHON_CMD="python3"
 elif command -v python &> /dev/null; then
    PYTHON_CMD="python"
 else
    echo -e "${RED}Error: Python 3.8+ is required but not found${NC}"
    echo -e "${YELLOW}Install from: https://www.python.org/downloads/${NC}"
    exit 1
 fi
 # Check Python version
 PYTHON_VERSION=$($PYTHON_CMD -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
 echo -e "${GREEN}✓ Found Python $PYTHON_VERSION${NC}"
 if [ "$(printf '%s\n' "3.8" "$PYTHON_VERSION" | sort -V | head -n1)" != "3.8" ]; then
    echo -e "${RED}Error: Python 3.8+ required, found $PYTHON_VERSION${NC}"
    exit 1
 fi
 # Check pip
 if ! $PYTHON_CMD -m pip --version &> /dev/null; then
    echo -e "${RED}Error: pip is required${NC}"
    echo -e "${YELLOW}Install pip: $PYTHON_CMD -m ensurepip${NC}"
    exit 1
 fi
 echo -e "${GREEN}✓ pip is available${NC}"
 # Check Docker
 echo -e "\n${BLUE}Checking Docker...${NC}"
 if ! command -v docker &> /dev/null; then
    echo -e "${YELLOW}⚠️  Docker not found (required for running servers)${NC}"
    echo -e "${YELLOW}Install from: https://docs.docker.com/get-docker/${NC}\n"
 else
    echo -e "${GREEN}✓ Docker is installed${NC}"
 fi
 # Check permissions
 USE_SUDO=""
 if [ ! -w "$INSTALL_DIR" ] || [ ! -w "/usr/local" ]; then
    echo -e "\n${YELLOW}⚠️  Root permission required for installation${NC}"
    USE_SUDO="sudo"
 fi
 # Create temp directory
 TMP_DIR="$(mktemp -d)"
 cd "$TMP_DIR"
 # Download only cnode_pkg from GitHub using sparse checkout
 echo -e "\n${BLUE}Downloading cnode package from GitHub...${NC}"
 if ! command -v git &> /dev/null; then
    echo -e "${RED}Error: git is required but not found${NC}"
    echo -e "${YELLOW}Install git and try again${NC}"
    rm -rf "$TMP_DIR"
    exit 1
 fi
 # Initialize sparse checkout
 git init -q
 git remote add origin "https://github.com/$GITHUB_REPO.git"
 git config core.sparseCheckout true
 # Only checkout the cnode_pkg directory
 echo "deploy/installer/cnode_pkg/*" > .git/info/sparse-checkout
 # Pull only the needed files
 if ! git pull -q --depth=1 origin "$BRANCH"; then
    echo -e "${RED}Error: Failed to download package${NC}"
    rm -rf "$TMP_DIR"
    exit 1
 fi
 if [ ! -d "deploy/installer/cnode_pkg" ]; then
    echo -e "${RED}Error: Package directory not found${NC}"
    rm -rf "$TMP_DIR"
    exit 1
 fi
 echo -e "${GREEN}✓ Package downloaded${NC}"
 REPO_DIR="."
 # Install Python dependencies
 echo -e "\n${BLUE}Installing Python dependencies...${NC}"
 $PYTHON_CMD -m pip install --quiet --user -r "$REPO_DIR/deploy/installer/cnode_pkg/requirements.txt" 2>/dev/null || \
 $PYTHON_CMD -m pip install --quiet --user --break-system-packages -r "$REPO_DIR/deploy/installer/cnode_pkg/requirements.txt" 2>/dev/null || {
    echo -e "${YELLOW}⚠️  Could not install dependencies with pip${NC}"
    echo -e "${YELLOW}Trying to continue anyway (dependencies may already be installed)${NC}"
 }
 echo -e "${GREEN}✓ Dependencies check complete${NC}"
 # Install cnode package
 echo -e "\n${BLUE}Installing cnode package...${NC}"
 $USE_SUDO mkdir -p "$LIB_DIR"
 $USE_SUDO cp -r "$REPO_DIR/deploy/installer/cnode_pkg" "$LIB_DIR/"
 echo -e "${GREEN}✓ Package installed to $LIB_DIR${NC}"
 # Create wrapper script
 echo -e "\n${BLUE}Creating cnode command...${NC}"
 $USE_SUDO tee "$INSTALL_DIR/cnode" > /dev/null << 'EOF'
 #!/usr/bin/env bash
 # Crawl4AI Node Manager (cnode) wrapper
 set -e
 # Find Python
 if command -v python3 &> /dev/null; then
    PYTHON_CMD="python3"
 elif command -v python &> /dev/null; then
    PYTHON_CMD="python"
 else
    echo "Error: Python 3.8+ required" >&2
    exit 1
 fi
 # Add cnode to Python path and run
 export PYTHONPATH="/usr/local/lib/cnode:$PYTHONPATH"
 exec $PYTHON_CMD -m cnode_pkg.cli "$@"
 EOF
 $USE_SUDO chmod +x "$INSTALL_DIR/cnode"
 echo -e "${GREEN}✓ cnode command created${NC}"
 # Cleanup
 rm -rf "$TMP_DIR"
 echo -e "\n${GREEN}✓ Installation complete${NC}"
 # Success message
 echo -e "\n${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}"
 echo -e "${GREEN}║              Installation Complete!                          ║${NC}"
 echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}\n"
 echo -e "${BLUE}cnode is now installed and ready!${NC}\n"
 echo -e "${YELLOW}Quick Start:${NC}"
 echo -e "  ${GREEN}cnode start${NC}                    # Start single server"
 echo -e "  ${GREEN}cnode start --replicas 5${NC}       # Start 5-replica cluster"
 echo -e "  ${GREEN}cnode status${NC}                   # Check status"
 echo -e "  ${GREEN}cnode logs -f${NC}                  # Follow logs"
 echo -e "  ${GREEN}cnode stop${NC}                     # Stop server"
 echo -e "\n${YELLOW}More help:${NC}"
 echo -e "  ${BLUE}cnode --help${NC}"
 echo -e "  ${BLUE}https://github.com/$GITHUB_REPO${NC}\n"
--- a/deploy/installer/sync-cnode.sh
+++ b/deploy/installer/sync-cnode.sh
@@ -1,33 +0,0 @@
 #!/bin/bash
 # Sync cnode source code to installer package
 # Run this before committing changes to cnode
 set -e
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 SOURCE_DIR="$SCRIPT_DIR/../docker"
 PKG_DIR="$SCRIPT_DIR/cnode_pkg"
 echo "🔄 Syncing cnode source to package..."
 # Copy CLI
 echo "  → Copying cnode_cli.py to cli.py"
 cp "$SOURCE_DIR/cnode_cli.py" "$PKG_DIR/cli.py"
 # Fix imports
 echo "  → Fixing imports (deploy.docker → cnode_pkg)"
 sed -i '' 's/from deploy\.docker\./from cnode_pkg./g' "$PKG_DIR/cli.py"
 # Copy server manager
 echo "  → Copying server_manager.py"
 cp "$SOURCE_DIR/server_manager.py" "$PKG_DIR/server_manager.py"
 echo "✅ Sync complete!"
 echo ""
 echo "Files updated:"
 echo "  • deploy/installer/cnode_pkg/cli.py"
 echo "  • deploy/installer/cnode_pkg/server_manager.py"
 echo ""
 echo "Next steps:"
 echo "  1. Test: cd deploy/installer && ./install-cnode.sh"
 echo "  2. Commit both source and package files"
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,18 +1,43 @@
 version: '3.8'
 # Shared configuration for all environments
 x-base-config: &base-config
  ports:
    - "11235:11235"  # Gunicorn port
  env_file:
    - .llm.env       # API keys (create from .llm.env.example)
  environment:
    - OPENAI_API_KEY=${OPENAI_API_KEY:-}
    - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-}
    - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
    - GROQ_API_KEY=${GROQ_API_KEY:-}
    - TOGETHER_API_KEY=${TOGETHER_API_KEY:-}
    - MISTRAL_API_KEY=${MISTRAL_API_KEY:-}
    - GEMINI_API_TOKEN=${GEMINI_API_TOKEN:-}
    - LLM_PROVIDER=${LLM_PROVIDER:-}  # Optional: Override default provider (e.g., "anthropic/claude-3-opus")
  volumes:
    - /dev/shm:/dev/shm  # Chromium performance
  deploy:
    resources:
      limits:
        memory: 4G
      reservations:
        memory: 1G
  restart: unless-stopped
  healthcheck:
    test: ["CMD", "curl", "-f", "http://localhost:11235/health"]
    interval: 30s
    timeout: 10s
    retries: 3
    start_period: 40s
  user: "appuser"
 services:
  redis:
    image: redis:alpine
    command: redis-server --appendonly yes
    volumes:
      - redis_data:/data
    networks:
      - crawl4ai_net
    restart: unless-stopped
  crawl4ai:
    # 1. Default: Pull multi-platform test image from Docker Hub
    # 2. Override with local image via: IMAGE=local-test docker compose up
    image: ${IMAGE:-unclecode/crawl4ai:${TAG:-latest}}
-
+    
    # Local build config (used with --build)
    build:
      context: .
@@ -20,58 +45,6 @@ services:
      args:
        INSTALL_TYPE: ${INSTALL_TYPE:-default}
        ENABLE_GPU: ${ENABLE_GPU:-false}
-
+    
-    # No ports exposed - access via nginx only
+    # Inherit shared config
-    env_file:
+    <<: *base-config
      - .llm.env
    environment:
      - OPENAI_API_KEY=${OPENAI_API_KEY:-}
      - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-}
      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
      - GROQ_API_KEY=${GROQ_API_KEY:-}
      - TOGETHER_API_KEY=${TOGETHER_API_KEY:-}
      - MISTRAL_API_KEY=${MISTRAL_API_KEY:-}
      - GEMINI_API_TOKEN=${GEMINI_API_TOKEN:-}
      - LLM_PROVIDER=${LLM_PROVIDER:-}
      - REDIS_HOST=redis
      - REDIS_PORT=6379
    volumes:
      - /dev/shm:/dev/shm  # Chromium performance
    deploy:
      replicas: 3  # Default to 3 replicas (can override with --scale)
      resources:
        limits:
          memory: 4G
        reservations:
          memory: 1G
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:11235/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    user: "appuser"
    depends_on:
      - redis
    networks:
      - crawl4ai_net
  nginx:
    image: nginx:alpine
    ports:
      - "11235:80"  # Expose port 11235 to host
    volumes:
      - ./crawl4ai/templates/nginx.conf.template:/etc/nginx/nginx.conf:ro
    depends_on:
      - crawl4ai
    networks:
      - crawl4ai_net
    restart: unless-stopped
 networks:
  crawl4ai_net:
    driver: bridge
 volumes:
  redis_data:
--- a/docs/md_v2/marketplace/app-detail.css
+++ b/docs/md_v2/marketplace/app-detail.css
@@ -278,12 +278,12 @@
 }
 .tab-content {
-    display: none;
+    display: none !important;
    padding: 2rem;
 }
 .tab-content.active {
-    display: block;
+    display: block !important;
 }
 /* Overview Layout */
--- a/docs/md_v2/marketplace/app-detail.html
+++ b/docs/md_v2/marketplace/app-detail.html
@@ -73,8 +73,8 @@
                <div class="tabs">
                    <button class="tab-btn active" data-tab="overview">Overview</button>
                    <button class="tab-btn" data-tab="integration">Integration</button>
-                    <button class="tab-btn" data-tab="docs">Documentation</button>
+                    <!-- <button class="tab-btn" data-tab="docs">Documentation</button>
-                    <button class="tab-btn" data-tab="support">Support</button>
+                    <button class="tab-btn" data-tab="support">Support</button> -->
                </div>
                <section id="overview-tab" class="tab-content active">
@@ -130,17 +130,15 @@
                <section id="integration-tab" class="tab-content">
                    <div class="integration-content" id="app-integration">
                        <!-- Integration guide markdown content will be rendered here -->
                    </div>
                </section>
-                <section id="docs-tab" class="tab-content">
+                <!-- <section id="docs-tab" class="tab-content">
                    <div class="docs-content" id="app-docs">
                        <!-- Documentation markdown content will be rendered here -->
                    </div>
-                </section>
+                </section> -->
-                <section id="support-tab" class="tab-content">
+                <!-- <section id="support-tab" class="tab-content">
                    <div class="docs-content">
                        <h2>Support</h2>
                        <div class="support-grid">
@@ -158,7 +156,7 @@
                            </div>
                        </div>
                    </div>
-                </section>
+                </section> -->
            </div>
        </main>
--- a/docs/md_v2/marketplace/app-detail.js
+++ b/docs/md_v2/marketplace/app-detail.js
@@ -112,7 +112,7 @@ class AppDetailPage {
        }
        // Contact
-        document.getElementById('app-contact').textContent = this.appData.contact_email || 'Not available';
+        document.getElementById('app-contact') && (document.getElementById('app-contact').textContent = this.appData.contact_email || 'Not available');
        // Sidebar info
        document.getElementById('sidebar-downloads').textContent = this.formatNumber(this.appData.downloads || 0);
@@ -263,18 +263,27 @@ class AppDetailPage {
    setupEventListeners() {
        // Tab switching
        const tabs = document.querySelectorAll('.tab-btn');
        tabs.forEach(tab => {
            tab.addEventListener('click', () => {
-                // Update active tab
+                // Update active tab button
                tabs.forEach(t => t.classList.remove('active'));
                tab.classList.add('active');
                // Show corresponding content
                const tabName = tab.dataset.tab;
-                document.querySelectorAll('.tab-content').forEach(content => {
+
                // Hide all tab contents
                const allTabContents = document.querySelectorAll('.tab-content');
                allTabContents.forEach(content => {
                    content.classList.remove('active');
                });
-                document.getElementById(`${tabName}-tab`).classList.add('active');
+
                // Show the selected tab content
                const targetTab = document.getElementById(`${tabName}-tab`);
                if (targetTab) {
                    targetTab.classList.add('active');
                }
            });
        });
    }
--- a/docs/md_v2/marketplace/backend/server.py
+++ b/docs/md_v2/marketplace/backend/server.py
@@ -471,13 +471,17 @@ async def delete_sponsor(sponsor_id: int):
 app.include_router(router)
 # Version info
 VERSION = "1.1.0"
 BUILD_DATE = "2025-10-26"
@app.get("/")
 async def root():
    """API info"""
    return {
        "name": "Crawl4AI Marketplace API",
-        "version": "1.0.0",
+        "version": VERSION,
        "build_date": BUILD_DATE,
        "endpoints": [
            "/marketplace/api/apps",
            "/marketplace/api/articles",
--- a/setup-hooks.sh
+++ b/setup-hooks.sh
@@ -1,24 +0,0 @@
 #!/bin/bash
 # Setup Git hooks for cnode auto-sync
 # Run this once after cloning the repo: ./setup-hooks.sh
 set -e
 echo "🔧 Setting up Git hooks..."
 # Configure Git to use .githooks directory
 git config core.hooksPath .githooks
 echo "✅ Git hooks configured!"
 echo ""
 echo "Hooks installed:"
 echo "  • pre-commit: Auto-syncs cnode source → package when committing"
 echo ""
 echo "What this means:"
 echo "  ✅ Edit deploy/docker/cnode_cli.py"
 echo "  ✅ Run: git add deploy/docker/cnode_cli.py"
 echo "  ✅ Run: git commit -m \"update cnode\""
 echo "  ✅ Hook automatically syncs to deploy/installer/cnode_pkg/"
 echo "  ✅ Synced files are auto-staged in the same commit"
 echo ""
 echo "You're all set! 🚀"
--- a/tests/cli/test_cli.py
+++ b/tests/cli/test_cli.py
@@ -65,14 +65,14 @@ class TestCLIBasics:
        assert 'Crawl4AI CLI' in result.output
    def test_examples(self, runner):
-        result = runner.invoke(cli, ['examples'])
+        result = runner.invoke(cli, ['--example'])
        assert result.exit_code == 0
        assert 'Examples' in result.output
    def test_missing_url(self, runner):
-        result = runner.invoke(cli, ['crawl'])
+        result = runner.invoke(cli)
        assert result.exit_code != 0
-        assert ('Missing argument' in result.output or 'required' in result.output.lower())
+        assert 'URL argument is required' in result.output
 class TestConfigParsing:
    def test_parse_key_values_basic(self):
@@ -101,19 +101,18 @@ class TestConfigLoading:
 class TestLLMConfig:
    def test_llm_config_creation(self, temp_config_dir, runner):
        def input_simulation(inputs):
-            return runner.invoke(cli, ['crawl', 'https://example.com', '-q', 'test question'],
+            return runner.invoke(cli, ['https://example.com', '-q', 'test question'], 
                               input='\n'.join(inputs))
 class TestCrawlingFeatures:
    def test_basic_crawl(self, runner):
-        result = runner.invoke(cli, ['crawl', 'https://example.com'])
+        result = runner.invoke(cli, ['https://example.com'])
        assert result.exit_code == 0
 class TestErrorHandling:
    def test_invalid_config_file(self, runner):
        result = runner.invoke(cli, [
            'crawl',
            'https://example.com',
            '--browser-config', 'nonexistent.yml'
        ])
@@ -123,9 +122,8 @@ class TestErrorHandling:
        invalid_schema = temp_config_dir / 'invalid_schema.json'
        with open(invalid_schema, 'w') as f:
            f.write('invalid json')
-
+            
        result = runner.invoke(cli, [
            'crawl',
            'https://example.com',
            '--schema', str(invalid_schema)
        ])
Author	SHA1	Message	Date
AHMET YILMAZ	05ec0535cd	#1564 : fix issue with _sig when using proxyConfig	2025-10-28 14:07:44 +08:00
AHMET YILMAZ	83aeb565ee	refactor(crawler_pool): enhance signature generation with adapter support and improve error handling	2025-10-28 14:02:43 +08:00
ntohidi	c0f1865287	feat(api): update marketplace version and build date in root endpoint response	2025-10-26 11:35:39 +01:00
ntohidi	46ef1116c4	fix(app-detail): enhance tab functionality, hide documentation and support tabs in marketplace	2025-10-26 11:21:29 +01:00