diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 00000000..ead3fc29 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,31 @@ +#!/bin/bash +# Pre-commit hook: Auto-sync cnode files when cnode source is modified + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +# Check if cnode source files are being committed +CNODE_FILES_CHANGED=$(git diff --cached --name-only | grep -E "deploy/docker/(cnode_cli|server_manager)\.py") + +if [ -n "$CNODE_FILES_CHANGED" ]; then + echo -e "${YELLOW}🔄 cnode source files modified, auto-syncing to package...${NC}" + + # Run sync script + if [ -f "deploy/installer/sync-cnode.sh" ]; then + bash deploy/installer/sync-cnode.sh + + # Stage the synced files + git add deploy/installer/cnode_pkg/cli.py + git add deploy/installer/cnode_pkg/server_manager.py + + echo -e "${GREEN}✅ cnode package synced and staged${NC}" + else + echo -e "${RED}❌ Error: sync-cnode.sh not found${NC}" + exit 1 + fi +fi + +exit 0 diff --git a/.gitignore b/.gitignore index 7fed1b79..b124a4f5 100644 --- a/.gitignore +++ b/.gitignore @@ -185,7 +185,8 @@ Crawl4AI.egg-info/ requirements0.txt a.txt -*.sh +# Ignore shell scripts globally, but allow test scripts +# *.sh .idea docs/examples/.chainlit/ docs/examples/.chainlit/* diff --git a/crawl4ai/cli.py b/crawl4ai/cli.py index 540fcde0..3397aae7 100644 --- a/crawl4ai/cli.py +++ b/crawl4ai/cli.py @@ -2,6 +2,8 @@ import click import os import sys import time +import subprocess +import shutil import humanize from typing import Dict, Any, Optional, List @@ -626,8 +628,73 @@ def cli(): # Register server command group (Docker orchestration) -from crawl4ai.server_cli import server_cmd -cli.add_command(server_cmd) +# Redirect to standalone 'cnode' CLI +@cli.command("server", context_settings=dict( + ignore_unknown_options=True, + allow_extra_args=True, + allow_interspersed_args=False +)) +@click.pass_context +def server_cmd(ctx): + """Manage Crawl4AI Docker server instances (deprecated - use 'cnode') + + This command has been moved to a standalone CLI called 'cnode'. + For new installations, use: + curl -sSL https://crawl4ai.com/deploy.sh | bash + + This redirect allows existing scripts to continue working. + + Available commands: start, stop, status, scale, logs + Use 'crwl server --help' for command-specific help. + """ + # Check if cnode is installed + cnode_path = shutil.which("cnode") + + # Get all the args (subcommand + options) + args = ctx.args + + if not cnode_path: + console.print(Panel( + "[yellow]The 'crwl server' command has been moved to a standalone CLI.[/yellow]\n\n" + "Please install 'cnode' (Crawl4AI Node Manager):\n" + "[cyan]curl -sSL https://crawl4ai.com/deploy.sh | bash[/cyan]\n\n" + "After installation, use:\n" + "[green]cnode [/green] instead of [dim]crwl server [/dim]\n\n" + "For backward compatibility, we're using the local version for now.", + title="Server Command Moved", + border_style="yellow" + )) + # Try to use local version + try: + import sys + # Add deploy/docker to path + deploy_path = str(Path(__file__).parent.parent / 'deploy' / 'docker') + if deploy_path not in sys.path: + sys.path.insert(0, deploy_path) + + from cnode_cli import cli as cnode_cli + + # Forward to cnode with the args + sys.argv = ['cnode'] + args + cnode_cli(standalone_mode=False) + sys.exit(0) + except SystemExit as e: + # Normal exit from click + sys.exit(e.code if hasattr(e, 'code') else 0) + except Exception as e: + console.print(f"[red]Error: Could not find cnode or local server CLI: {e}[/red]") + console.print(f"[dim]Details: {e}[/dim]") + import traceback + console.print(f"[dim]{traceback.format_exc()}[/dim]") + sys.exit(1) + + # cnode is installed - forward everything to it + try: + result = subprocess.run([cnode_path] + args, check=False) + sys.exit(result.returncode) + except Exception as e: + console.print(f"[red]Error running cnode: {e}[/red]") + sys.exit(1) @cli.group("browser") @@ -1467,9 +1534,15 @@ def default(url: str, example: bool, browser_config: str, crawler_config: str, f def main(): import sys - if len(sys.argv) < 2 or sys.argv[1] not in cli.commands: + # Don't auto-insert 'crawl' if the command is recognized + if len(sys.argv) >= 2 and sys.argv[1] in cli.commands: + cli() + elif len(sys.argv) < 2: + cli() + else: + # Unknown command - insert 'crawl' for backward compat sys.argv.insert(1, "crawl") - cli() + cli() if __name__ == "__main__": main() \ No newline at end of file diff --git a/deploy/docker/__init__.py b/deploy/docker/__init__.py new file mode 100644 index 00000000..674f5965 --- /dev/null +++ b/deploy/docker/__init__.py @@ -0,0 +1 @@ +# Deploy docker module diff --git a/deploy/docker/cnode_cli.py b/deploy/docker/cnode_cli.py new file mode 100644 index 00000000..f955a2d9 --- /dev/null +++ b/deploy/docker/cnode_cli.py @@ -0,0 +1,492 @@ +""" +Crawl4AI Server CLI Commands + +Provides `cnode` command group for Docker orchestration. +""" + +import click +import anyio +from rich.console import Console +from rich.table import Table +from rich.panel import Panel +from rich.prompt import Confirm + +from deploy.docker.server_manager import ServerManager + + +console = Console() + + +@click.group() +def cli(): + """Manage Crawl4AI Docker server instances + + \b + One-command deployment with automatic scaling: + • Single container for development (N=1) + • Docker Swarm for production with built-in load balancing (N>1) + • Docker Compose + Nginx as fallback (N>1) + + \b + Examples: + cnode start # Single container on port 11235 + cnode start --replicas 3 # Auto-detect Swarm or Compose + cnode start -r 5 --port 8080 # 5 replicas on custom port + cnode status # Check current deployment + cnode scale 10 # Scale to 10 replicas + cnode stop # Stop and cleanup + """ + pass + + +@cli.command("start") +@click.option( + "--replicas", "-r", + type=int, + default=1, + help="Number of container replicas (default: 1)" +) +@click.option( + "--mode", + type=click.Choice(["auto", "single", "swarm", "compose"]), + default="auto", + help="Deployment mode (default: auto-detect)" +) +@click.option( + "--port", "-p", + type=int, + default=11235, + help="External port to expose (default: 11235)" +) +@click.option( + "--env-file", + type=click.Path(exists=True), + help="Path to environment file" +) +@click.option( + "--image", + default="unclecode/crawl4ai:latest", + help="Docker image to use (default: unclecode/crawl4ai:latest)" +) +def start_cmd(replicas: int, mode: str, port: int, env_file: str, image: str): + """Start Crawl4AI server with automatic orchestration. + + Deployment modes: + - auto: Automatically choose best mode (default) + - single: Single container (N=1 only) + - swarm: Docker Swarm with built-in load balancing + - compose: Docker Compose + Nginx reverse proxy + + The server will: + 1. Check if Docker is running + 2. Validate port availability + 3. Pull image if needed + 4. Start container(s) with health checks + 5. Save state for management + + Examples: + # Development: single container + cnode start + + # Production: 5 replicas with Swarm + cnode start --replicas 5 + + # Custom configuration + cnode start -r 3 --port 8080 --env-file .env.prod + """ + manager = ServerManager() + + console.print(Panel( + f"[cyan]Starting Crawl4AI Server[/cyan]\n\n" + f"Replicas: [yellow]{replicas}[/yellow]\n" + f"Mode: [yellow]{mode}[/yellow]\n" + f"Port: [yellow]{port}[/yellow]\n" + f"Image: [yellow]{image}[/yellow]", + title="Server Start", + border_style="cyan" + )) + + with console.status("[cyan]Starting server..."): + async def _start(): + return await manager.start( + replicas=replicas, + mode=mode, + port=port, + env_file=env_file, + image=image + ) + result = anyio.run(_start) + + if result["success"]: + console.print(Panel( + f"[green]✓ Server started successfully![/green]\n\n" + f"Mode: [cyan]{result.get('state_data', {}).get('mode', mode)}[/cyan]\n" + f"URL: [bold]http://localhost:{port}[/bold]\n" + f"Health: [bold]http://localhost:{port}/health[/bold]\n" + f"Monitor: [bold]http://localhost:{port}/monitor[/bold]", + title="Server Running", + border_style="green" + )) + else: + error_msg = result.get("error", result.get("message", "Unknown error")) + console.print(Panel( + f"[red]✗ Failed to start server[/red]\n\n" + f"{error_msg}", + title="Error", + border_style="red" + )) + + if "already running" in error_msg.lower(): + console.print("\n[yellow]Hint: Use 'cnode status' to check current deployment[/yellow]") + console.print("[yellow] Use 'cnode stop' to stop existing server[/yellow]") + + +@cli.command("status") +def status_cmd(): + """Show current server status and deployment info. + + Displays: + - Running state (up/down) + - Deployment mode (single/swarm/compose) + - Number of replicas + - Port mapping + - Uptime + - Image version + + Example: + cnode status + """ + manager = ServerManager() + + async def _status(): + return await manager.status() + result = anyio.run(_status) + + if result["running"]: + table = Table(title="Crawl4AI Server Status", border_style="green") + table.add_column("Property", style="cyan") + table.add_column("Value", style="green") + + table.add_row("Status", "🟢 Running") + table.add_row("Mode", result["mode"]) + table.add_row("Replicas", str(result.get("replicas", 1))) + table.add_row("Port", str(result.get("port", 11235))) + table.add_row("Image", result.get("image", "unknown")) + table.add_row("Uptime", result.get("uptime", "unknown")) + table.add_row("Started", result.get("started_at", "unknown")) + + console.print(table) + console.print(f"\n[green]✓ Server is healthy[/green]") + console.print(f"[dim]Access: http://localhost:{result.get('port', 11235)}[/dim]") + else: + console.print(Panel( + f"[yellow]No server is currently running[/yellow]\n\n" + f"Use 'cnode start' to launch a server", + title="Server Status", + border_style="yellow" + )) + + +@cli.command("stop") +@click.option( + "--remove-volumes", + is_flag=True, + help="Remove associated volumes (WARNING: deletes data)" +) +def stop_cmd(remove_volumes: bool): + """Stop running Crawl4AI server and cleanup resources. + + This will: + 1. Stop all running containers/services + 2. Remove containers + 3. Optionally remove volumes (--remove-volumes) + 4. Clean up state files + + WARNING: Use --remove-volumes with caution as it will delete + persistent data including Redis databases and logs. + + Examples: + # Stop server, keep volumes + cnode stop + + # Stop and remove all data + cnode stop --remove-volumes + """ + manager = ServerManager() + + # Confirm if removing volumes + if remove_volumes: + if not Confirm.ask( + "[red]⚠️ This will delete all server data including Redis databases. Continue?[/red]" + ): + console.print("[yellow]Cancelled[/yellow]") + return + + with console.status("[cyan]Stopping server..."): + async def _stop(): + return await manager.stop(remove_volumes=remove_volumes) + result = anyio.run(_stop) + + if result["success"]: + console.print(Panel( + f"[green]✓ Server stopped successfully[/green]\n\n" + f"{result.get('message', 'All resources cleaned up')}", + title="Server Stopped", + border_style="green" + )) + else: + console.print(Panel( + f"[red]✗ Error stopping server[/red]\n\n" + f"{result.get('error', result.get('message', 'Unknown error'))}", + title="Error", + border_style="red" + )) + + +@cli.command("scale") +@click.argument("replicas", type=int) +def scale_cmd(replicas: int): + """Scale server to specified number of replicas. + + Only works with Swarm or Compose modes. Single container + mode cannot be scaled (must stop and restart with --replicas). + + Scaling is live and does not require downtime. The load + balancer will automatically distribute traffic to new replicas. + + Examples: + # Scale up to 10 replicas + cnode scale 10 + + # Scale down to 2 replicas + cnode scale 2 + + # Scale to 1 (minimum) + cnode scale 1 + """ + if replicas < 1: + console.print("[red]Error: Replicas must be at least 1[/red]") + return + + manager = ServerManager() + + with console.status(f"[cyan]Scaling to {replicas} replicas..."): + async def _scale(): + return await manager.scale(replicas=replicas) + result = anyio.run(_scale) + + if result["success"]: + console.print(Panel( + f"[green]✓ Scaled successfully[/green]\n\n" + f"New replica count: [bold]{replicas}[/bold]\n" + f"Mode: [cyan]{result.get('mode')}[/cyan]", + title="Scaling Complete", + border_style="green" + )) + else: + error_msg = result.get("error", result.get("message", "Unknown error")) + console.print(Panel( + f"[red]✗ Scaling failed[/red]\n\n" + f"{error_msg}", + title="Error", + border_style="red" + )) + + if "single container" in error_msg.lower(): + console.print("\n[yellow]Hint: For single container mode:[/yellow]") + console.print("[yellow] 1. cnode stop[/yellow]") + console.print(f"[yellow] 2. cnode start --replicas {replicas}[/yellow]") + + +@cli.command("logs") +@click.option( + "--follow", "-f", + is_flag=True, + help="Follow log output (like tail -f)" +) +@click.option( + "--tail", + type=int, + default=100, + help="Number of lines to show (default: 100)" +) +def logs_cmd(follow: bool, tail: int): + """View server logs. + + Shows logs from running containers/services. Use --follow + to stream logs in real-time. + + Examples: + # Show last 100 lines + cnode logs + + # Show last 500 lines + cnode logs --tail 500 + + # Follow logs in real-time + cnode logs --follow + + # Combine options + cnode logs -f --tail 50 + """ + manager = ServerManager() + + async def _logs(): + return await manager.logs(follow=follow, tail=tail) + output = anyio.run(_logs) + console.print(output) + + +@cli.command("cleanup") +@click.option( + "--force", + is_flag=True, + help="Force cleanup even if state file doesn't exist" +) +def cleanup_cmd(force: bool): + """Force cleanup of all Crawl4AI Docker resources. + + Stops and removes all containers, networks, and optionally volumes. + Useful when server is stuck or state is corrupted. + + Examples: + # Clean up everything + cnode cleanup + + # Force cleanup (ignore state file) + cnode cleanup --force + """ + manager = ServerManager() + + console.print(Panel( + f"[yellow]⚠️ Cleaning up Crawl4AI Docker resources[/yellow]\n\n" + f"This will stop and remove:\n" + f"- All Crawl4AI containers\n" + f"- Nginx load balancer\n" + f"- Redis instance\n" + f"- Docker networks\n" + f"- State files", + title="Cleanup", + border_style="yellow" + )) + + if not force and not Confirm.ask("[yellow]Continue with cleanup?[/yellow]"): + console.print("[yellow]Cancelled[/yellow]") + return + + with console.status("[cyan]Cleaning up resources..."): + async def _cleanup(): + return await manager.cleanup(force=force) + result = anyio.run(_cleanup) + + if result["success"]: + console.print(Panel( + f"[green]✓ Cleanup completed successfully[/green]\n\n" + f"Removed: {result.get('removed', 0)} containers\n" + f"{result.get('message', 'All resources cleaned up')}", + title="Cleanup Complete", + border_style="green" + )) + else: + console.print(Panel( + f"[yellow]⚠️ Partial cleanup[/yellow]\n\n" + f"{result.get('message', 'Some resources may still exist')}", + title="Cleanup Status", + border_style="yellow" + )) + + +@cli.command("restart") +@click.option( + "--replicas", "-r", + type=int, + help="New replica count (optional)" +) +def restart_cmd(replicas: int): + """Restart server (stop then start with same config). + + Preserves existing configuration unless overridden with options. + Useful for applying image updates or recovering from errors. + + Examples: + # Restart with same configuration + cnode restart + + # Restart and change replica count + cnode restart --replicas 5 + """ + manager = ServerManager() + + # Get current state + async def _get_status(): + return await manager.status() + current = anyio.run(_get_status) + + if not current["running"]: + console.print("[yellow]No server is running. Use 'cnode start' instead.[/yellow]") + return + + # Extract current config + current_replicas = current.get("replicas", 1) + current_port = current.get("port", 11235) + current_image = current.get("image", "unclecode/crawl4ai:latest") + current_mode = current.get("mode", "auto") + + # Override with CLI args + new_replicas = replicas if replicas is not None else current_replicas + + console.print(Panel( + f"[cyan]Restarting Crawl4AI Server[/cyan]\n\n" + f"Replicas: [yellow]{current_replicas}[/yellow] → [green]{new_replicas}[/green]\n" + f"Port: [yellow]{current_port}[/yellow]\n" + f"Mode: [yellow]{current_mode}[/yellow]", + title="Server Restart", + border_style="cyan" + )) + + # Stop current + with console.status("[cyan]Stopping current server..."): + async def _stop_server(): + return await manager.stop(remove_volumes=False) + stop_result = anyio.run(_stop_server) + + if not stop_result["success"]: + console.print(f"[red]Failed to stop server: {stop_result.get('error')}[/red]") + return + + # Start new + with console.status("[cyan]Starting server..."): + async def _start_server(): + return await manager.start( + replicas=new_replicas, + mode="auto", + port=current_port, + image=current_image + ) + start_result = anyio.run(_start_server) + + if start_result["success"]: + console.print(Panel( + f"[green]✓ Server restarted successfully![/green]\n\n" + f"URL: [bold]http://localhost:{current_port}[/bold]", + title="Restart Complete", + border_style="green" + )) + else: + console.print(Panel( + f"[red]✗ Failed to restart server[/red]\n\n" + f"{start_result.get('error', 'Unknown error')}", + title="Error", + border_style="red" + )) + + +def main(): + """Entry point for cnode CLI""" + cli() + + +if __name__ == "__main__": + main() + +# Test comment diff --git a/deploy/docker/server_manager.py b/deploy/docker/server_manager.py new file mode 100644 index 00000000..1a29a706 --- /dev/null +++ b/deploy/docker/server_manager.py @@ -0,0 +1,1154 @@ +""" +Crawl4AI Docker Server Manager + +Orchestrates single-node Docker deployments with automatic scaling: +- Single container (N=1) +- Docker Swarm (N>1, if available) +- Docker Compose + Nginx (N>1, fallback) +""" + +import json +import subprocess +import time +import re +import os +from pathlib import Path +from typing import Dict, Optional, Literal +from datetime import datetime +import socket + + +ServerMode = Literal["single", "swarm", "compose"] + + +# ========== Input Validation Functions ========== + +def validate_docker_image(image: str) -> bool: + """Validate Docker image name format. + + Allows: registry.com/namespace/repo:tag + Format: [registry/][namespace/]repo[:tag][@digest] + + Args: + image: Docker image string + + Returns: + True if valid, False otherwise + """ + if not image or not isinstance(image, str): + return False + + # Length check + if len(image) > 256: + return False + + # Basic pattern: alphanumeric, dots, slashes, colons, dashes, underscores + # No shell metacharacters allowed + pattern = r'^[a-zA-Z0-9.\-/:_@]+$' + if not re.match(pattern, image): + return False + + # Additional safety: no consecutive special chars that could be exploited + if '..' in image or '//' in image: + return False + + return True + + +def validate_port(port: int) -> bool: + """Validate port number is in valid range. + + Args: + port: Port number + + Returns: + True if valid (1-65535), False otherwise + """ + return isinstance(port, int) and 1 <= port <= 65535 + + +def validate_env_file(path: str) -> bool: + """Validate environment file path exists and is readable. + + Args: + path: File path to validate + + Returns: + True if file exists and is readable, False otherwise + """ + if not path or not isinstance(path, str): + return False + + try: + file_path = Path(path).resolve() + return file_path.exists() and file_path.is_file() and os.access(file_path, os.R_OK) + except Exception: + return False + + +def validate_replicas(replicas: int) -> bool: + """Validate replica count is in reasonable range. + + Args: + replicas: Number of replicas + + Returns: + True if valid (1-100), False otherwise + """ + return isinstance(replicas, int) and 1 <= replicas <= 100 + + +class ServerManager: + """Manages Crawl4AI Docker server lifecycle and orchestration.""" + + def __init__(self): + self.state_dir = Path.home() / ".crawl4ai" / "server" + self.state_file = self.state_dir / "state.json" + self.compose_file = self.state_dir / "docker-compose.yml" + self.nginx_conf = self.state_dir / "nginx.conf" + self.state_dir.mkdir(parents=True, exist_ok=True) + + # ========== Public API ========== + + async def start( + self, + replicas: int = 1, + mode: str = "auto", + port: int = 11235, + env_file: Optional[str] = None, + image: str = "unclecode/crawl4ai:latest", + **kwargs + ) -> Dict: + """Start Crawl4AI server with specified configuration. + + Args: + replicas: Number of container replicas (default: 1) + mode: Deployment mode - 'auto', 'single', 'swarm', or 'compose' + port: External port to expose (default: 11235) + env_file: Path to environment file + image: Docker image to use + **kwargs: Additional docker run arguments + + Returns: + Dict with status and deployment info + """ + # Check if already running + state = self._load_state() + if state: + return { + "success": False, + "message": "Server already running", + "current_state": state + } + + # Validate Docker is available + if not self._is_docker_available(): + return { + "success": False, + "error": "Docker daemon not running. Please start Docker first." + } + + # Check port availability + if not self._is_port_available(port): + return { + "success": False, + "error": f"Port {port} is already in use" + } + + # Detect deployment mode + detected_mode = self._detect_mode(replicas, mode) + + # Ensure image is available + if not self._ensure_image(image): + return { + "success": False, + "error": f"Failed to pull image {image}" + } + + # Start based on mode + if detected_mode == "single": + result = self._start_single(port, env_file, image, **kwargs) + elif detected_mode == "swarm": + result = self._start_swarm(replicas, port, env_file, image, **kwargs) + elif detected_mode == "compose": + result = self._start_compose(replicas, port, env_file, image, **kwargs) + else: + return { + "success": False, + "error": f"Unknown mode: {detected_mode}" + } + + if result["success"]: + # Save state + self._save_state({ + "mode": detected_mode, + "replicas": replicas, + "port": port, + "image": image, + "env_file": env_file, + "started_at": datetime.now().isoformat(), + **result.get("state_data", {}) + }) + + return result + + async def status(self) -> Dict: + """Get current server status.""" + state = self._load_state() + + if not state: + return { + "running": False, + "message": "No server is currently running" + } + + mode = state["mode"] + + # Check actual container status + if mode == "single": + running = self._check_container_running(state.get("container_id")) + elif mode == "swarm": + running = self._check_service_running(state.get("service_name")) + elif mode == "compose": + running = self._check_compose_running(state.get("compose_project")) + else: + running = False + + if not running: + # State file exists but containers are gone - clean up + self._clear_state() + return { + "running": False, + "message": "State file exists but containers stopped externally" + } + + return { + "running": True, + "mode": mode, + "replicas": state.get("replicas", 1), + "port": state.get("port", 11235), + "image": state.get("image"), + "started_at": state.get("started_at"), + "uptime": self._calculate_uptime(state.get("started_at")) + } + + async def stop(self, remove_volumes: bool = False) -> Dict: + """Stop running server. + + Args: + remove_volumes: Remove associated volumes + + Returns: + Dict with stop status + """ + state = self._load_state() + + if not state: + return { + "success": False, + "message": "No server is running" + } + + mode = state["mode"] + + try: + if mode == "single": + self._stop_single(state.get("container_id"), remove_volumes) + elif mode == "swarm": + self._stop_swarm(state.get("service_name")) + elif mode == "compose": + self._stop_compose(state.get("compose_project"), remove_volumes) + + self._clear_state() + + return { + "success": True, + "message": f"Server stopped ({mode} mode)" + } + except Exception as e: + return { + "success": False, + "error": str(e) + } + + async def cleanup(self, force: bool = False) -> Dict: + """Force cleanup of all Crawl4AI Docker resources. + + Args: + force: Force cleanup even if state file doesn't exist + + Returns: + Dict with cleanup status + """ + import logging + logger = logging.getLogger(__name__) + + removed_count = 0 + messages = [] + + try: + # Try to stop via state file first + if not force: + state = self._load_state() + if state: + stop_result = await self.stop(remove_volumes=True) + if stop_result["success"]: + return { + "success": True, + "removed": 1, + "message": "Stopped via state file" + } + + # Force cleanup - find and remove all Crawl4AI resources + logger.info("Force cleanup: removing all Crawl4AI Docker resources") + + # Remove all crawl4ai containers + try: + result = subprocess.run( + ["docker", "ps", "-a", "--filter", "name=crawl4ai", "--format", "{{.ID}}"], + capture_output=True, + text=True, + timeout=10 + ) + container_ids = result.stdout.strip().split('\n') + container_ids = [cid for cid in container_ids if cid] + + for cid in container_ids: + subprocess.run(["docker", "rm", "-f", cid], capture_output=True, timeout=10) + removed_count += 1 + messages.append(f"Removed {len(container_ids)} crawl4ai containers") + except Exception as e: + logger.warning(f"Error removing containers: {e}") + + # Remove nginx containers + try: + result = subprocess.run( + ["docker", "ps", "-a", "--filter", "name=nginx", "--format", "{{.ID}}"], + capture_output=True, + text=True, + timeout=10 + ) + nginx_ids = result.stdout.strip().split('\n') + nginx_ids = [nid for nid in nginx_ids if nid] + + for nid in nginx_ids: + subprocess.run(["docker", "rm", "-f", nid], capture_output=True, timeout=10) + removed_count += len(nginx_ids) + if nginx_ids: + messages.append(f"Removed {len(nginx_ids)} nginx containers") + except Exception as e: + logger.warning(f"Error removing nginx: {e}") + + # Remove redis containers + try: + result = subprocess.run( + ["docker", "ps", "-a", "--filter", "name=redis", "--format", "{{.ID}}"], + capture_output=True, + text=True, + timeout=10 + ) + redis_ids = result.stdout.strip().split('\n') + redis_ids = [rid for rid in redis_ids if rid] + + for rid in redis_ids: + subprocess.run(["docker", "rm", "-f", rid], capture_output=True, timeout=10) + removed_count += len(redis_ids) + if redis_ids: + messages.append(f"Removed {len(redis_ids)} redis containers") + except Exception as e: + logger.warning(f"Error removing redis: {e}") + + # Clean up compose projects + for project in ["crawl4ai", "fix-docker"]: + try: + subprocess.run( + ["docker", "compose", "-p", project, "down", "-v"], + capture_output=True, + timeout=30, + cwd=str(self.state_dir) + ) + messages.append(f"Cleaned compose project: {project}") + except Exception: + pass + + # Remove networks + try: + subprocess.run(["docker", "network", "prune", "-f"], capture_output=True, timeout=10) + messages.append("Pruned networks") + except Exception as e: + logger.warning(f"Error pruning networks: {e}") + + # Clear state file + self._clear_state() + messages.append("Cleared state file") + + return { + "success": True, + "removed": removed_count, + "message": "; ".join(messages) + } + + except Exception as e: + logger.error(f"Cleanup error: {e}") + return { + "success": False, + "message": f"Cleanup failed: {str(e)}" + } + + async def scale(self, replicas: int) -> Dict: + """Scale server to specified replica count. + + Args: + replicas: Target number of replicas + + Returns: + Dict with scaling status + """ + state = self._load_state() + + if not state: + return { + "success": False, + "message": "No server is running" + } + + mode = state["mode"] + + if mode == "single": + return { + "success": False, + "error": "Cannot scale single container mode. Use 'crwl server stop' then 'crwl server start --replicas N'" + } + + try: + if mode == "swarm": + self._scale_swarm(state["service_name"], replicas) + elif mode == "compose": + self._scale_compose(state["compose_project"], replicas) + + # Update state + state["replicas"] = replicas + self._save_state(state) + + return { + "success": True, + "message": f"Scaled to {replicas} replicas", + "mode": mode + } + except Exception as e: + return { + "success": False, + "error": str(e) + } + + async def logs(self, follow: bool = False, tail: int = 100) -> str: + """Get server logs. + + Args: + follow: Follow log output + tail: Number of lines to show + + Returns: + Log output as string + """ + state = self._load_state() + + if not state: + return "No server is running" + + mode = state["mode"] + + try: + if mode == "single": + return self._logs_single(state["container_id"], follow, tail) + elif mode == "swarm": + return self._logs_swarm(state["service_name"], follow, tail) + elif mode == "compose": + return self._logs_compose(state["compose_project"], follow, tail) + except Exception as e: + return f"Error getting logs: {e}" + + # ========== Mode Detection ========== + + def _detect_mode(self, replicas: int, mode: str) -> ServerMode: + """Detect deployment mode based on replicas and user preference.""" + if mode != "auto": + return mode + + if replicas == 1: + return "single" + + # N>1: prefer Swarm if available, fallback to Compose + if self._is_swarm_available(): + return "swarm" + + return "compose" + + def _is_swarm_available(self) -> bool: + """Check if Docker Swarm is initialized and available.""" + try: + result = subprocess.run( + ["docker", "info", "--format", "{{.Swarm.LocalNodeState}}"], + capture_output=True, + text=True, + timeout=5 + ) + return result.stdout.strip() == "active" + except Exception: + return False + + def _is_docker_available(self) -> bool: + """Check if Docker daemon is running.""" + try: + subprocess.run( + ["docker", "ps"], + capture_output=True, + timeout=5, + check=True + ) + return True + except Exception: + return False + + def _is_port_available(self, port: int) -> bool: + """Check if port is available for binding.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(('0.0.0.0', port)) + return True + except OSError: + return False + + def _ensure_image(self, image: str) -> bool: + """Ensure Docker image is available locally, pull if needed.""" + try: + # Check if image exists locally + result = subprocess.run( + ["docker", "image", "inspect", image], + capture_output=True, + timeout=5 + ) + + if result.returncode == 0: + return True + + # Determine if this looks like a registry image + # Registry images have format: [registry/][namespace/]repository[:tag] + # Examples: unclecode/crawl4ai:latest, docker.io/library/nginx:latest + # Local-only: crawl4ai-local:latest, my-image:v1 + + # If it has a dot in the first part (before any slash), it's likely a registry + # Or if it has a slash, it's likely registry/namespace/repo format + parts = image.split("/") + is_registry_image = ( + len(parts) > 1 and # Has slash + "." not in parts[0] and # First part isn't a domain (localhost.localdomain) + not parts[0].startswith("localhost") # Not localhost registry + ) + + if not is_registry_image: + return False # Local image doesn't exist + + # Try to pull from registry + subprocess.run( + ["docker", "pull", image], + capture_output=True, + check=True, + timeout=300 + ) + return True + except Exception: + return False + + # ========== Single Container Mode ========== + + def _start_single(self, port: int, env_file: Optional[str], image: str, **kwargs) -> Dict: + """Start single container with docker run.""" + # Validate inputs to prevent injection attacks + if not validate_port(port): + return { + "success": False, + "error": f"Invalid port number: {port}. Must be between 1-65535." + } + + if not validate_docker_image(image): + return { + "success": False, + "error": f"Invalid Docker image format: {image}" + } + + if env_file and not validate_env_file(env_file): + return { + "success": False, + "error": f"Environment file not found or not readable: {env_file}" + } + + cmd = [ + "docker", "run", + "-d", # Detached + "--name", "crawl4ai_server", + "-p", f"{port}:11235", + "--shm-size=1g", # Important for browser + ] + + if env_file: + # Use absolute path to prevent path traversal + abs_env_file = str(Path(env_file).resolve()) + cmd.extend(["--env-file", abs_env_file]) + + # Whitelist allowed Docker flags to prevent privilege escalation + allowed_flags = {"--memory", "--cpus", "--restart", "--network"} + for key, value in kwargs.items(): + if key in allowed_flags: + cmd.append(key) + if value is not True: # Handle boolean flags + cmd.append(str(value)) + else: + # Log ignored flags for debugging + import logging + logging.warning(f"Ignoring non-whitelisted Docker flag: {key}") + + cmd.append(image) + + try: + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + container_id = result.stdout.strip() + + # Wait for health check + if self._wait_for_health(f"http://localhost:{port}/health"): + return { + "success": True, + "message": f"Server started on port {port}", + "state_data": {"container_id": container_id} + } + else: + # Cleanup failed container + subprocess.run(["docker", "rm", "-f", container_id], capture_output=True) + return { + "success": False, + "error": "Container started but health check failed" + } + except subprocess.CalledProcessError as e: + return { + "success": False, + "error": f"Failed to start container: {e.stderr}" + } + + def _stop_single(self, container_id: str, remove_volumes: bool): + """Stop single container.""" + cmd = ["docker", "rm", "-f"] + if remove_volumes: + cmd.append("-v") + cmd.append(container_id) + subprocess.run(cmd, check=True) + + def _check_container_running(self, container_id: str) -> bool: + """Check if container is running.""" + if not container_id: + return False + try: + result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", container_id], + capture_output=True, + text=True, + timeout=5 + ) + return result.stdout.strip() == "true" + except Exception: + return False + + def _logs_single(self, container_id: str, follow: bool, tail: int) -> str: + """Get logs from single container.""" + cmd = ["docker", "logs", "--tail", str(tail)] + if follow: + cmd.append("-f") + cmd.append(container_id) + + result = subprocess.run(cmd, capture_output=True, text=True) + return result.stdout + + # ========== Swarm Mode ========== + + def _start_swarm(self, replicas: int, port: int, env_file: Optional[str], image: str, **kwargs) -> Dict: + """Start service in Swarm mode.""" + # Validate inputs to prevent injection attacks + if not validate_replicas(replicas): + return { + "success": False, + "error": f"Invalid replica count: {replicas}. Must be between 1-100." + } + + if not validate_port(port): + return { + "success": False, + "error": f"Invalid port number: {port}. Must be between 1-65535." + } + + if not validate_docker_image(image): + return { + "success": False, + "error": f"Invalid Docker image format: {image}" + } + + if env_file and not validate_env_file(env_file): + return { + "success": False, + "error": f"Environment file not found or not readable: {env_file}" + } + + service_name = "crawl4ai" # Static name (safe) + + # Initialize swarm if needed + if not self._is_swarm_available(): + init_result = self._init_swarm() + if not init_result: + return { + "success": False, + "error": "Failed to initialize Docker Swarm. Use 'docker swarm init' manually." + } + + cmd = [ + "docker", "service", "create", + "--name", service_name, + "--replicas", str(replicas), + "--publish", f"{port}:11235", + "--mount", "type=tmpfs,target=/dev/shm,tmpfs-size=1g", + "--limit-memory", "4G", + ] + + if env_file: + # Use absolute path to prevent path traversal + abs_env_file = str(Path(env_file).resolve()) + cmd.extend(["--env-file", abs_env_file]) + + cmd.append(image) + + try: + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + service_id = result.stdout.strip() + + # Wait for service to be ready (check replicas) + if self._wait_for_service(service_name, replicas): + return { + "success": True, + "message": f"Swarm service started with {replicas} replicas", + "state_data": { + "service_name": service_name, + "service_id": service_id + } + } + else: + # Cleanup failed service + subprocess.run(["docker", "service", "rm", service_name], capture_output=True) + return { + "success": False, + "error": "Service created but replicas failed to start" + } + except subprocess.CalledProcessError as e: + return { + "success": False, + "error": f"Failed to create Swarm service: {e.stderr}" + } + + def _init_swarm(self) -> bool: + """Initialize Docker Swarm if not already initialized.""" + try: + result = subprocess.run( + ["docker", "swarm", "init"], + capture_output=True, + text=True, + timeout=10 + ) + return result.returncode == 0 + except Exception: + return False + + def _wait_for_service(self, service_name: str, expected_replicas: int, timeout: int = 60) -> bool: + """Wait for Swarm service replicas to be running.""" + import time + start = time.time() + + while time.time() - start < timeout: + try: + result = subprocess.run( + ["docker", "service", "ls", "--filter", f"name={service_name}", "--format", "{{.Replicas}}"], + capture_output=True, + text=True, + timeout=5 + ) + + if result.returncode == 0: + # Format is "2/3" (running/desired) + replicas_str = result.stdout.strip() + if "/" in replicas_str: + running, desired = replicas_str.split("/") + if int(running) == expected_replicas and int(desired) == expected_replicas: + return True + + time.sleep(2) + except Exception: + time.sleep(2) + + return False + + def _stop_swarm(self, service_name: str): + """Stop Swarm service.""" + subprocess.run( + ["docker", "service", "rm", service_name], + check=True, + capture_output=True + ) + + def _scale_swarm(self, service_name: str, replicas: int): + """Scale Swarm service.""" + subprocess.run( + ["docker", "service", "scale", f"{service_name}={replicas}"], + check=True, + capture_output=True + ) + + def _check_service_running(self, service_name: str) -> bool: + """Check if Swarm service is running.""" + if not service_name: + return False + try: + result = subprocess.run( + ["docker", "service", "ls", "--filter", f"name={service_name}", "--format", "{{.Name}}"], + capture_output=True, + text=True, + timeout=5 + ) + return service_name in result.stdout + except Exception: + return False + + def _logs_swarm(self, service_name: str, follow: bool, tail: int) -> str: + """Get logs from Swarm service.""" + cmd = ["docker", "service", "logs", "--tail", str(tail)] + if follow: + cmd.append("-f") + cmd.append(service_name) + + result = subprocess.run(cmd, capture_output=True, text=True) + return result.stdout + + # ========== Compose Mode ========== + + def _start_compose(self, replicas: int, port: int, env_file: Optional[str], image: str, **kwargs) -> Dict: + """Start with Docker Compose + Nginx.""" + # Validate inputs to prevent injection attacks + if not validate_replicas(replicas): + return { + "success": False, + "error": f"Invalid replica count: {replicas}. Must be between 1-100." + } + + if not validate_port(port): + return { + "success": False, + "error": f"Invalid port number: {port}. Must be between 1-65535." + } + + if not validate_docker_image(image): + return { + "success": False, + "error": f"Invalid Docker image format: {image}" + } + + if env_file and not validate_env_file(env_file): + return { + "success": False, + "error": f"Environment file not found or not readable: {env_file}" + } + + project_name = "crawl4ai" # Static name (safe) + + # Generate compose and nginx config files + try: + self._generate_compose_file(replicas, port, env_file or "", image) + self._generate_nginx_config() + except Exception as e: + return { + "success": False, + "error": f"Failed to generate config files: {e}" + } + + # Start compose stack - use absolute path for compose file + cmd = [ + "docker", "compose", + "-f", str(self.compose_file.resolve()), + "-p", project_name, + "up", "-d", + "--scale", f"crawl4ai={replicas}" + ] + + try: + result = subprocess.run(cmd, capture_output=True, text=True, check=True, cwd=str(self.state_dir)) + + # Wait for services to be healthy + if self._wait_for_compose_healthy(project_name, timeout=60): + return { + "success": True, + "message": f"Compose stack started with {replicas} replicas", + "state_data": { + "compose_project": project_name + } + } + else: + # Cleanup failed deployment + subprocess.run( + ["docker", "compose", "-f", str(self.compose_file), "-p", project_name, "down"], + capture_output=True, + cwd=str(self.state_dir) + ) + return { + "success": False, + "error": "Compose stack started but health checks failed" + } + except subprocess.CalledProcessError as e: + return { + "success": False, + "error": f"Failed to start Compose stack: {e.stderr}" + } + + def _generate_compose_file(self, replicas: int, port: int, env_file: str, image: str): + """Generate docker-compose.yml from template with validation.""" + import os + + # Get template path - check if we're in the package or dev environment + template_path = Path(__file__).parent / "templates" / "docker-compose.template.yml" + + if not template_path.exists(): + raise FileNotFoundError( + f"Docker Compose template not found: {template_path}\n" + f"Please ensure crawl4ai package is correctly installed.\n" + f"Try: pip install --force-reinstall crawl4ai" + ) + + try: + with open(template_path) as f: + template = f.read() + except IOError as e: + raise RuntimeError(f"Failed to read template {template_path}: {e}") + + # Validate template has required placeholders + required_vars = {"${IMAGE}", "${REPLICAS}", "${PORT}", "${NGINX_CONF}"} + missing = required_vars - set(re.findall(r'\$\{[A-Z_]+\}', template)) + if missing: + raise ValueError(f"Template missing required variables: {missing}") + + # Substitute variables + content = template.replace("${IMAGE}", image) + content = content.replace("${REPLICAS}", str(replicas)) + content = content.replace("${PORT}", str(port)) + content = content.replace("${NGINX_CONF}", str(self.nginx_conf)) + + # Verify no unsubstituted variables remain + remaining = re.findall(r'\$\{[A-Z_]+\}', content) + if remaining: + import logging + logging.warning(f"Unsubstituted variables in template: {remaining}") + + try: + with open(self.compose_file, "w") as f: + f.write(content) + except IOError as e: + raise RuntimeError(f"Failed to write compose file {self.compose_file}: {e}") + + def _generate_nginx_config(self): + """Generate nginx.conf from template with validation.""" + template_path = Path(__file__).parent / "templates" / "nginx.conf.template" + + if not template_path.exists(): + raise FileNotFoundError( + f"Nginx template not found: {template_path}\n" + f"Please ensure crawl4ai package is correctly installed.\n" + f"Try: pip install --force-reinstall crawl4ai" + ) + + try: + with open(template_path) as f: + content = f.read() + except IOError as e: + raise RuntimeError(f"Failed to read nginx template {template_path}: {e}") + + # Nginx template doesn't need variable substitution currently + try: + with open(self.nginx_conf, "w") as f: + f.write(content) + except IOError as e: + raise RuntimeError(f"Failed to write nginx config {self.nginx_conf}: {e}") + + def _wait_for_compose_healthy(self, project: str, timeout: int = 60) -> bool: + """Wait for Compose services to be healthy.""" + import time + start = time.time() + + while time.time() - start < timeout: + try: + # Check if nginx service is running (it depends on crawl4ai) + result = subprocess.run( + ["docker", "compose", "-f", str(self.compose_file), "-p", project, "ps", "--format", "json"], + capture_output=True, + text=True, + timeout=5, + cwd=str(self.state_dir) + ) + + if result.returncode == 0 and result.stdout: + import json + services = [json.loads(line) for line in result.stdout.strip().split('\n') if line] + + # Check if nginx is running (implies crawl4ai instances are up) + nginx_running = any( + s.get("Service") == "nginx" and s.get("State") == "running" + for s in services + ) + + if nginx_running: + return True + + time.sleep(2) + except Exception: + time.sleep(2) + + return False + + def _stop_compose(self, project: str, remove_volumes: bool): + """Stop Compose stack.""" + cmd = ["docker", "compose", "-f", str(self.compose_file), "-p", project, "down"] + if remove_volumes: + cmd.append("-v") + + subprocess.run(cmd, check=True, capture_output=True, cwd=str(self.state_dir)) + + def _scale_compose(self, project: str, replicas: int): + """Scale Compose service.""" + subprocess.run( + ["docker", "compose", "-f", str(self.compose_file), "-p", project, "up", "-d", "--scale", f"crawl4ai={replicas}", "--no-recreate"], + check=True, + capture_output=True, + cwd=str(self.state_dir) + ) + + def _check_compose_running(self, project: str) -> bool: + """Check if Compose stack is running.""" + if not project or not self.compose_file.exists(): + return False + try: + result = subprocess.run( + ["docker", "compose", "-f", str(self.compose_file), "-p", project, "ps", "-q"], + capture_output=True, + text=True, + timeout=5, + cwd=str(self.state_dir) + ) + # If there are any container IDs, the stack is running + return bool(result.stdout.strip()) + except Exception: + return False + + def _logs_compose(self, project: str, follow: bool, tail: int) -> str: + """Get logs from Compose stack.""" + cmd = ["docker", "compose", "-f", str(self.compose_file), "-p", project, "logs", "--tail", str(tail)] + if follow: + cmd.append("-f") + + result = subprocess.run(cmd, capture_output=True, text=True, cwd=str(self.state_dir)) + return result.stdout + + # ========== State Management ========== + + def _save_state(self, state: Dict): + """Persist server state to disk with atomic write and file locking.""" + import fcntl + + self.state_dir.mkdir(parents=True, exist_ok=True) + + # Atomic write with exclusive lock + temp_file = self.state_file.with_suffix('.tmp') + try: + with open(temp_file, 'w') as f: + fcntl.flock(f.fileno(), fcntl.LOCK_EX) # Exclusive lock + json.dump(state, f, indent=2) + f.flush() + os.fsync(f.fileno()) # Force write to disk + fcntl.flock(f.fileno(), fcntl.LOCK_UN) # Unlock + + # Atomic rename + temp_file.replace(self.state_file) + except Exception as e: + # Cleanup temp file on error + temp_file.unlink(missing_ok=True) + raise RuntimeError(f"Failed to save state: {e}") + + def _load_state(self) -> Optional[Dict]: + """Load server state from disk with file locking.""" + import fcntl + + if not self.state_file.exists(): + return None + + try: + with open(self.state_file) as f: + fcntl.flock(f.fileno(), fcntl.LOCK_SH) # Shared lock (read) + state = json.load(f) + fcntl.flock(f.fileno(), fcntl.LOCK_UN) # Unlock + return state + except (json.JSONDecodeError, IOError) as e: + # Log and remove corrupted state file + import logging + logging.error(f"Corrupted state file, removing: {e}") + self.state_file.unlink(missing_ok=True) + return None + + def _clear_state(self): + """Remove state file with locking.""" + import fcntl + + if self.state_file.exists(): + try: + # Acquire lock before deletion to prevent race + with open(self.state_file, 'r') as f: + fcntl.flock(f.fileno(), fcntl.LOCK_EX) + # Lock acquired, now delete + self.state_file.unlink(missing_ok=True) + except Exception: + # If lock fails, force delete anyway + self.state_file.unlink(missing_ok=True) + + # ========== Helpers ========== + + def _wait_for_health(self, url: str, timeout: int = 30) -> bool: + """Wait for health endpoint to respond.""" + import urllib.request + + start = time.time() + while time.time() - start < timeout: + try: + urllib.request.urlopen(url, timeout=2) + return True + except Exception: + time.sleep(1) + return False + + def _calculate_uptime(self, started_at: str) -> str: + """Calculate uptime from ISO timestamp.""" + if not started_at: + return "unknown" + + try: + start = datetime.fromisoformat(started_at) + delta = datetime.now() - start + + hours = delta.seconds // 3600 + minutes = (delta.seconds % 3600) // 60 + + if delta.days > 0: + return f"{delta.days}d {hours}h {minutes}m" + elif hours > 0: + return f"{hours}h {minutes}m" + else: + return f"{minutes}m" + except Exception: + return "unknown" diff --git a/deploy/installer/README.md b/deploy/installer/README.md new file mode 100644 index 00000000..1f2c1da0 --- /dev/null +++ b/deploy/installer/README.md @@ -0,0 +1,345 @@ +# Crawl4AI Node Manager (cnode) - Installation & Distribution + +This directory contains the standalone `cnode` package and installation scripts for managing Crawl4AI Docker server instances. + +## Overview + +`cnode` is a fast, lightweight CLI tool for managing Crawl4AI Docker servers. It provides: +- One-command deployment with automatic scaling +- Single container for development (N=1) +- Docker Swarm for production with built-in load balancing (N>1) +- Docker Compose + Nginx as fallback (N>1) + +## Directory Structure + +``` +deploy/installer/ +├── README.md # This file +├── cnode_pkg/ # Standalone Python package +│ ├── __init__.py # Package marker +│ ├── cli.py # CLI interface (commands) +│ ├── server_manager.py # Docker orchestration logic +│ └── requirements.txt # Python dependencies +├── install-cnode.sh # Local installation script +├── deploy.sh # Remote installation script (for users) +└── releases/ # Release artifacts for distribution +``` + +## Installation + +### For Users (Remote Installation) + +Users can install `cnode` directly from the web: + +```bash +# Install from GitHub/website +curl -sSL https://crawl4ai.com/install-cnode.sh | bash + +# Or with wget +wget -qO- https://crawl4ai.com/install-cnode.sh | bash +``` + +### For Local Testing + +Test the installation locally: + +```bash +cd deploy/installer +./install-cnode.sh +``` + +## Package Contents + +### `cnode_pkg/` - Python Package + +This is a self-contained Python package with: + +- **`cli.py`**: Click-based CLI with all commands (start, stop, status, scale, logs, cleanup, restart) +- **`server_manager.py`**: Core Docker orchestration logic +- **`requirements.txt`**: Dependencies (click, rich, anyio, pyyaml) +- **`__init__.py`**: Package initialization + +### Installation Script + +**`install-cnode.sh`** does the following: +1. Checks for Python 3.8+ and pip +2. Checks for Docker (warns if not found) +3. Installs Python dependencies +4. Copies `cnode_pkg/` to `/usr/local/lib/cnode/` +5. Creates wrapper script at `/usr/local/bin/cnode` +6. Verifies installation + +### Wrapper Script + +Created at `/usr/local/bin/cnode`: + +```bash +#!/usr/bin/env bash +set -e + +# Find Python +if command -v python3 &> /dev/null; then + PYTHON_CMD="python3" +elif command -v python &> /dev/null; then + PYTHON_CMD="python" +else + echo "Error: Python 3.8+ required" >&2 + exit 1 +fi + +# Run cnode +export PYTHONPATH="/usr/local/lib/cnode:$PYTHONPATH" +exec $PYTHON_CMD -m cnode_pkg.cli "$@" +``` + +## Performance + +**Blazing Fast Startup:** +- **~0.1 seconds** to launch +- 49x faster than compiled binary alternatives +- Minimal overhead, maximum responsiveness + +## Requirements + +### User Requirements +- Python 3.8 or higher +- pip (Python package manager) +- Docker (for running servers) + +### Dependencies (Auto-installed) +- click >= 8.0.0 (CLI framework) +- rich >= 13.0.0 (Terminal formatting) +- anyio >= 3.0.0 (Async I/O) +- pyyaml >= 6.0.0 (YAML parsing) + +## Usage + +After installation: + +```bash +# Quick start +cnode start # Single container on port 11235 +cnode start --replicas 5 # 5-replica cluster +cnode status # Check server status +cnode logs -f # Follow logs +cnode scale 10 # Scale to 10 replicas +cnode stop # Stop server + +# Get help +cnode --help +cnode start --help +``` + +## Development Workflow + +### Making Changes + +1. **Edit source code** in `deploy/docker/`: + ```bash + vim deploy/docker/cnode_cli.py + vim deploy/docker/server_manager.py + ``` + +2. **Update package** by copying to installer: + ```bash + # Copy CLI + cp deploy/docker/cnode_cli.py deploy/installer/cnode_pkg/cli.py + + # Fix imports (deploy.docker → cnode_pkg) + sed -i 's/from deploy\.docker\./from cnode_pkg./g' deploy/installer/cnode_pkg/cli.py + + # Copy server manager + cp deploy/docker/server_manager.py deploy/installer/cnode_pkg/server_manager.py + ``` + +3. **Test locally**: + ```bash + cd deploy/installer + ./install-cnode.sh + cnode --help + ``` + +4. **Commit both**: + ```bash + git add deploy/docker/cnode_cli.py + git add deploy/installer/cnode_pkg/cli.py + git commit -m "Update cnode: [description]" + ``` + +### Creating a Release + +1. **Tag the release**: + ```bash + git tag -a v1.0.0 -m "Release v1.0.0" + git push origin v1.0.0 + ``` + +2. **Package for distribution**: + ```bash + cd deploy/installer + tar -czf releases/cnode-v1.0.0.tar.gz cnode_pkg/ install-cnode.sh + ``` + +3. **Create GitHub release**: + ```bash + gh release create v1.0.0 \ + releases/cnode-v1.0.0.tar.gz \ + --title "cnode v1.0.0" \ + --notes "Release notes here" + ``` + +4. **Update deployment script** (if needed): + - Update `deploy.sh` with new version/URL + - Upload to hosting (e.g., `https://crawl4ai.com/install-cnode.sh`) + +## Deployment + +### Remote Installation Script + +The `deploy.sh` script is meant to be hosted at a public URL for user installation: + +```bash +# Upload to your server +scp deploy.sh user@crawl4ai.com:/var/www/html/install-cnode.sh + +# Or use GitHub raw URL +https://raw.githubusercontent.com/unclecode/crawl4ai/main/deploy/installer/deploy.sh +``` + +Users can then install with: +```bash +curl -sSL https://crawl4ai.com/install-cnode.sh | bash +``` + +## Backward Compatibility + +The main Crawl4AI CLI (`crwl`) includes a redirect for backward compatibility: + +```bash +# These work identically: +crwl server start --replicas 3 +cnode start --replicas 3 + +# All subcommands redirect: +crwl server status → cnode status +crwl server stop → cnode stop +crwl server scale 5 → cnode scale 5 +crwl server logs -f → cnode logs -f +``` + +This ensures existing scripts continue working while users migrate to `cnode`. + +## Uninstallation + +To remove cnode: + +```bash +# Remove command +sudo rm /usr/local/bin/cnode + +# Remove package +sudo rm -rf /usr/local/lib/cnode + +# (Optional) Uninstall dependencies +pip uninstall click rich anyio pyyaml +``` + +## Troubleshooting + +### Python Not Found +```bash +# Install Python 3.8+ +# macOS: brew install python3 +# Ubuntu: sudo apt install python3 python3-pip +# RHEL/CentOS: sudo yum install python3 python3-pip +``` + +### Permission Denied +```bash +# Run installer with sudo +sudo ./install-cnode.sh + +# Or change install location +INSTALL_DIR=$HOME/.local/bin ./install-cnode.sh +``` + +### Command Not Found After Install +```bash +# Add to PATH in ~/.bashrc or ~/.zshrc +export PATH="/usr/local/bin:$PATH" + +# Reload shell +source ~/.bashrc # or source ~/.zshrc +``` + +### Dependencies Install Failed +```bash +# Install manually +pip install --user click rich anyio pyyaml + +# Or with break-system-packages (if needed) +pip install --user --break-system-packages click rich anyio pyyaml +``` + +### Docker Not Running +```bash +# macOS: Start Docker Desktop +# Linux: sudo systemctl start docker + +# Check Docker +docker --version +docker ps +``` + +## Architecture + +### Component Flow + +``` +User runs: cnode start + ↓ +/usr/local/bin/cnode (wrapper script) + ↓ +Finds python3 executable + ↓ +Sets PYTHONPATH=/usr/local/lib/cnode + ↓ +python3 -m cnode_pkg.cli start + ↓ +cli.py → start_cmd() + ↓ +server_manager.py → ServerManager.start() + ↓ +Docker orchestration (single/swarm/compose) + ↓ +Server running! +``` + +### Why Python Wrapper vs Binary? + +We chose a Python wrapper over compiled binaries (PyInstaller) because: + +| Metric | Python Wrapper | PyInstaller Binary | +|--------|---------------|-------------------| +| Startup time | **0.1s** | 4.7s | +| Size | ~50KB wrapper | 8.8MB | +| Updates | Easy (just copy files) | Rebuild required | +| Dependencies | Python 3.8+ | None | +| Platform | Any with Python | OS-specific builds | + +Since users running Crawl4AI already have Python, the wrapper is the clear winner. + +## Support + +For issues or questions: +- GitHub Issues: https://github.com/unclecode/crawl4ai/issues +- Documentation: https://docs.crawl4ai.com +- Discord: https://discord.gg/crawl4ai + +## Version History + +- **v1.0.0**: Initial release with Python wrapper approach + - Fast startup (~0.1s) + - Supports single container, Docker Swarm, and Compose modes + - Auto-scaling and load balancing + - Real-time monitoring and logs diff --git a/deploy/installer/cnode_pkg/__init__.py b/deploy/installer/cnode_pkg/__init__.py new file mode 100644 index 00000000..d233738f --- /dev/null +++ b/deploy/installer/cnode_pkg/__init__.py @@ -0,0 +1,5 @@ +""" +Crawl4AI Node Manager (cnode) - Docker server orchestration CLI +""" + +__version__ = "1.0.0" diff --git a/deploy/installer/cnode_pkg/cli.py b/deploy/installer/cnode_pkg/cli.py new file mode 100644 index 00000000..9f88ac84 --- /dev/null +++ b/deploy/installer/cnode_pkg/cli.py @@ -0,0 +1,492 @@ +""" +Crawl4AI Server CLI Commands + +Provides `cnode` command group for Docker orchestration. +""" + +import click +import anyio +from rich.console import Console +from rich.table import Table +from rich.panel import Panel +from rich.prompt import Confirm + +from cnode_pkg.server_manager import ServerManager + + +console = Console() + + +@click.group() +def cli(): + """Manage Crawl4AI Docker server instances + + \b + One-command deployment with automatic scaling: + • Single container for development (N=1) + • Docker Swarm for production with built-in load balancing (N>1) + • Docker Compose + Nginx as fallback (N>1) + + \b + Examples: + cnode start # Single container on port 11235 + cnode start --replicas 3 # Auto-detect Swarm or Compose + cnode start -r 5 --port 8080 # 5 replicas on custom port + cnode status # Check current deployment + cnode scale 10 # Scale to 10 replicas + cnode stop # Stop and cleanup + """ + pass + + +@cli.command("start") +@click.option( + "--replicas", "-r", + type=int, + default=1, + help="Number of container replicas (default: 1)" +) +@click.option( + "--mode", + type=click.Choice(["auto", "single", "swarm", "compose"]), + default="auto", + help="Deployment mode (default: auto-detect)" +) +@click.option( + "--port", "-p", + type=int, + default=11235, + help="External port to expose (default: 11235)" +) +@click.option( + "--env-file", + type=click.Path(exists=True), + help="Path to environment file" +) +@click.option( + "--image", + default="unclecode/crawl4ai:latest", + help="Docker image to use (default: unclecode/crawl4ai:latest)" +) +def start_cmd(replicas: int, mode: str, port: int, env_file: str, image: str): + """Start Crawl4AI server with automatic orchestration. + + Deployment modes: + - auto: Automatically choose best mode (default) + - single: Single container (N=1 only) + - swarm: Docker Swarm with built-in load balancing + - compose: Docker Compose + Nginx reverse proxy + + The server will: + 1. Check if Docker is running + 2. Validate port availability + 3. Pull image if needed + 4. Start container(s) with health checks + 5. Save state for management + + Examples: + # Development: single container + cnode start + + # Production: 5 replicas with Swarm + cnode start --replicas 5 + + # Custom configuration + cnode start -r 3 --port 8080 --env-file .env.prod + """ + manager = ServerManager() + + console.print(Panel( + f"[cyan]Starting Crawl4AI Server[/cyan]\n\n" + f"Replicas: [yellow]{replicas}[/yellow]\n" + f"Mode: [yellow]{mode}[/yellow]\n" + f"Port: [yellow]{port}[/yellow]\n" + f"Image: [yellow]{image}[/yellow]", + title="Server Start", + border_style="cyan" + )) + + with console.status("[cyan]Starting server..."): + async def _start(): + return await manager.start( + replicas=replicas, + mode=mode, + port=port, + env_file=env_file, + image=image + ) + result = anyio.run(_start) + + if result["success"]: + console.print(Panel( + f"[green]✓ Server started successfully![/green]\n\n" + f"Mode: [cyan]{result.get('state_data', {}).get('mode', mode)}[/cyan]\n" + f"URL: [bold]http://localhost:{port}[/bold]\n" + f"Health: [bold]http://localhost:{port}/health[/bold]\n" + f"Monitor: [bold]http://localhost:{port}/monitor[/bold]", + title="Server Running", + border_style="green" + )) + else: + error_msg = result.get("error", result.get("message", "Unknown error")) + console.print(Panel( + f"[red]✗ Failed to start server[/red]\n\n" + f"{error_msg}", + title="Error", + border_style="red" + )) + + if "already running" in error_msg.lower(): + console.print("\n[yellow]Hint: Use 'cnode status' to check current deployment[/yellow]") + console.print("[yellow] Use 'cnode stop' to stop existing server[/yellow]") + + +@cli.command("status") +def status_cmd(): + """Show current server status and deployment info. + + Displays: + - Running state (up/down) + - Deployment mode (single/swarm/compose) + - Number of replicas + - Port mapping + - Uptime + - Image version + + Example: + cnode status + """ + manager = ServerManager() + + async def _status(): + return await manager.status() + result = anyio.run(_status) + + if result["running"]: + table = Table(title="Crawl4AI Server Status", border_style="green") + table.add_column("Property", style="cyan") + table.add_column("Value", style="green") + + table.add_row("Status", "🟢 Running") + table.add_row("Mode", result["mode"]) + table.add_row("Replicas", str(result.get("replicas", 1))) + table.add_row("Port", str(result.get("port", 11235))) + table.add_row("Image", result.get("image", "unknown")) + table.add_row("Uptime", result.get("uptime", "unknown")) + table.add_row("Started", result.get("started_at", "unknown")) + + console.print(table) + console.print(f"\n[green]✓ Server is healthy[/green]") + console.print(f"[dim]Access: http://localhost:{result.get('port', 11235)}[/dim]") + else: + console.print(Panel( + f"[yellow]No server is currently running[/yellow]\n\n" + f"Use 'cnode start' to launch a server", + title="Server Status", + border_style="yellow" + )) + + +@cli.command("stop") +@click.option( + "--remove-volumes", + is_flag=True, + help="Remove associated volumes (WARNING: deletes data)" +) +def stop_cmd(remove_volumes: bool): + """Stop running Crawl4AI server and cleanup resources. + + This will: + 1. Stop all running containers/services + 2. Remove containers + 3. Optionally remove volumes (--remove-volumes) + 4. Clean up state files + + WARNING: Use --remove-volumes with caution as it will delete + persistent data including Redis databases and logs. + + Examples: + # Stop server, keep volumes + cnode stop + + # Stop and remove all data + cnode stop --remove-volumes + """ + manager = ServerManager() + + # Confirm if removing volumes + if remove_volumes: + if not Confirm.ask( + "[red]⚠️ This will delete all server data including Redis databases. Continue?[/red]" + ): + console.print("[yellow]Cancelled[/yellow]") + return + + with console.status("[cyan]Stopping server..."): + async def _stop(): + return await manager.stop(remove_volumes=remove_volumes) + result = anyio.run(_stop) + + if result["success"]: + console.print(Panel( + f"[green]✓ Server stopped successfully[/green]\n\n" + f"{result.get('message', 'All resources cleaned up')}", + title="Server Stopped", + border_style="green" + )) + else: + console.print(Panel( + f"[red]✗ Error stopping server[/red]\n\n" + f"{result.get('error', result.get('message', 'Unknown error'))}", + title="Error", + border_style="red" + )) + + +@cli.command("scale") +@click.argument("replicas", type=int) +def scale_cmd(replicas: int): + """Scale server to specified number of replicas. + + Only works with Swarm or Compose modes. Single container + mode cannot be scaled (must stop and restart with --replicas). + + Scaling is live and does not require downtime. The load + balancer will automatically distribute traffic to new replicas. + + Examples: + # Scale up to 10 replicas + cnode scale 10 + + # Scale down to 2 replicas + cnode scale 2 + + # Scale to 1 (minimum) + cnode scale 1 + """ + if replicas < 1: + console.print("[red]Error: Replicas must be at least 1[/red]") + return + + manager = ServerManager() + + with console.status(f"[cyan]Scaling to {replicas} replicas..."): + async def _scale(): + return await manager.scale(replicas=replicas) + result = anyio.run(_scale) + + if result["success"]: + console.print(Panel( + f"[green]✓ Scaled successfully[/green]\n\n" + f"New replica count: [bold]{replicas}[/bold]\n" + f"Mode: [cyan]{result.get('mode')}[/cyan]", + title="Scaling Complete", + border_style="green" + )) + else: + error_msg = result.get("error", result.get("message", "Unknown error")) + console.print(Panel( + f"[red]✗ Scaling failed[/red]\n\n" + f"{error_msg}", + title="Error", + border_style="red" + )) + + if "single container" in error_msg.lower(): + console.print("\n[yellow]Hint: For single container mode:[/yellow]") + console.print("[yellow] 1. cnode stop[/yellow]") + console.print(f"[yellow] 2. cnode start --replicas {replicas}[/yellow]") + + +@cli.command("logs") +@click.option( + "--follow", "-f", + is_flag=True, + help="Follow log output (like tail -f)" +) +@click.option( + "--tail", + type=int, + default=100, + help="Number of lines to show (default: 100)" +) +def logs_cmd(follow: bool, tail: int): + """View server logs. + + Shows logs from running containers/services. Use --follow + to stream logs in real-time. + + Examples: + # Show last 100 lines + cnode logs + + # Show last 500 lines + cnode logs --tail 500 + + # Follow logs in real-time + cnode logs --follow + + # Combine options + cnode logs -f --tail 50 + """ + manager = ServerManager() + + async def _logs(): + return await manager.logs(follow=follow, tail=tail) + output = anyio.run(_logs) + console.print(output) + + +@cli.command("cleanup") +@click.option( + "--force", + is_flag=True, + help="Force cleanup even if state file doesn't exist" +) +def cleanup_cmd(force: bool): + """Force cleanup of all Crawl4AI Docker resources. + + Stops and removes all containers, networks, and optionally volumes. + Useful when server is stuck or state is corrupted. + + Examples: + # Clean up everything + cnode cleanup + + # Force cleanup (ignore state file) + cnode cleanup --force + """ + manager = ServerManager() + + console.print(Panel( + f"[yellow]⚠️ Cleaning up Crawl4AI Docker resources[/yellow]\n\n" + f"This will stop and remove:\n" + f"- All Crawl4AI containers\n" + f"- Nginx load balancer\n" + f"- Redis instance\n" + f"- Docker networks\n" + f"- State files", + title="Cleanup", + border_style="yellow" + )) + + if not force and not Confirm.ask("[yellow]Continue with cleanup?[/yellow]"): + console.print("[yellow]Cancelled[/yellow]") + return + + with console.status("[cyan]Cleaning up resources..."): + async def _cleanup(): + return await manager.cleanup(force=force) + result = anyio.run(_cleanup) + + if result["success"]: + console.print(Panel( + f"[green]✓ Cleanup completed successfully[/green]\n\n" + f"Removed: {result.get('removed', 0)} containers\n" + f"{result.get('message', 'All resources cleaned up')}", + title="Cleanup Complete", + border_style="green" + )) + else: + console.print(Panel( + f"[yellow]⚠️ Partial cleanup[/yellow]\n\n" + f"{result.get('message', 'Some resources may still exist')}", + title="Cleanup Status", + border_style="yellow" + )) + + +@cli.command("restart") +@click.option( + "--replicas", "-r", + type=int, + help="New replica count (optional)" +) +def restart_cmd(replicas: int): + """Restart server (stop then start with same config). + + Preserves existing configuration unless overridden with options. + Useful for applying image updates or recovering from errors. + + Examples: + # Restart with same configuration + cnode restart + + # Restart and change replica count + cnode restart --replicas 5 + """ + manager = ServerManager() + + # Get current state + async def _get_status(): + return await manager.status() + current = anyio.run(_get_status) + + if not current["running"]: + console.print("[yellow]No server is running. Use 'cnode start' instead.[/yellow]") + return + + # Extract current config + current_replicas = current.get("replicas", 1) + current_port = current.get("port", 11235) + current_image = current.get("image", "unclecode/crawl4ai:latest") + current_mode = current.get("mode", "auto") + + # Override with CLI args + new_replicas = replicas if replicas is not None else current_replicas + + console.print(Panel( + f"[cyan]Restarting Crawl4AI Server[/cyan]\n\n" + f"Replicas: [yellow]{current_replicas}[/yellow] → [green]{new_replicas}[/green]\n" + f"Port: [yellow]{current_port}[/yellow]\n" + f"Mode: [yellow]{current_mode}[/yellow]", + title="Server Restart", + border_style="cyan" + )) + + # Stop current + with console.status("[cyan]Stopping current server..."): + async def _stop_server(): + return await manager.stop(remove_volumes=False) + stop_result = anyio.run(_stop_server) + + if not stop_result["success"]: + console.print(f"[red]Failed to stop server: {stop_result.get('error')}[/red]") + return + + # Start new + with console.status("[cyan]Starting server..."): + async def _start_server(): + return await manager.start( + replicas=new_replicas, + mode="auto", + port=current_port, + image=current_image + ) + start_result = anyio.run(_start_server) + + if start_result["success"]: + console.print(Panel( + f"[green]✓ Server restarted successfully![/green]\n\n" + f"URL: [bold]http://localhost:{current_port}[/bold]", + title="Restart Complete", + border_style="green" + )) + else: + console.print(Panel( + f"[red]✗ Failed to restart server[/red]\n\n" + f"{start_result.get('error', 'Unknown error')}", + title="Error", + border_style="red" + )) + + +def main(): + """Entry point for cnode CLI""" + cli() + + +if __name__ == "__main__": + main() + +# Test comment diff --git a/deploy/installer/cnode_pkg/requirements.txt b/deploy/installer/cnode_pkg/requirements.txt new file mode 100644 index 00000000..7329b2d3 --- /dev/null +++ b/deploy/installer/cnode_pkg/requirements.txt @@ -0,0 +1,4 @@ +click>=8.0.0 +rich>=13.0.0 +anyio>=3.0.0 +pyyaml>=6.0.0 diff --git a/deploy/installer/cnode_pkg/server_manager.py b/deploy/installer/cnode_pkg/server_manager.py new file mode 100644 index 00000000..1a29a706 --- /dev/null +++ b/deploy/installer/cnode_pkg/server_manager.py @@ -0,0 +1,1154 @@ +""" +Crawl4AI Docker Server Manager + +Orchestrates single-node Docker deployments with automatic scaling: +- Single container (N=1) +- Docker Swarm (N>1, if available) +- Docker Compose + Nginx (N>1, fallback) +""" + +import json +import subprocess +import time +import re +import os +from pathlib import Path +from typing import Dict, Optional, Literal +from datetime import datetime +import socket + + +ServerMode = Literal["single", "swarm", "compose"] + + +# ========== Input Validation Functions ========== + +def validate_docker_image(image: str) -> bool: + """Validate Docker image name format. + + Allows: registry.com/namespace/repo:tag + Format: [registry/][namespace/]repo[:tag][@digest] + + Args: + image: Docker image string + + Returns: + True if valid, False otherwise + """ + if not image or not isinstance(image, str): + return False + + # Length check + if len(image) > 256: + return False + + # Basic pattern: alphanumeric, dots, slashes, colons, dashes, underscores + # No shell metacharacters allowed + pattern = r'^[a-zA-Z0-9.\-/:_@]+$' + if not re.match(pattern, image): + return False + + # Additional safety: no consecutive special chars that could be exploited + if '..' in image or '//' in image: + return False + + return True + + +def validate_port(port: int) -> bool: + """Validate port number is in valid range. + + Args: + port: Port number + + Returns: + True if valid (1-65535), False otherwise + """ + return isinstance(port, int) and 1 <= port <= 65535 + + +def validate_env_file(path: str) -> bool: + """Validate environment file path exists and is readable. + + Args: + path: File path to validate + + Returns: + True if file exists and is readable, False otherwise + """ + if not path or not isinstance(path, str): + return False + + try: + file_path = Path(path).resolve() + return file_path.exists() and file_path.is_file() and os.access(file_path, os.R_OK) + except Exception: + return False + + +def validate_replicas(replicas: int) -> bool: + """Validate replica count is in reasonable range. + + Args: + replicas: Number of replicas + + Returns: + True if valid (1-100), False otherwise + """ + return isinstance(replicas, int) and 1 <= replicas <= 100 + + +class ServerManager: + """Manages Crawl4AI Docker server lifecycle and orchestration.""" + + def __init__(self): + self.state_dir = Path.home() / ".crawl4ai" / "server" + self.state_file = self.state_dir / "state.json" + self.compose_file = self.state_dir / "docker-compose.yml" + self.nginx_conf = self.state_dir / "nginx.conf" + self.state_dir.mkdir(parents=True, exist_ok=True) + + # ========== Public API ========== + + async def start( + self, + replicas: int = 1, + mode: str = "auto", + port: int = 11235, + env_file: Optional[str] = None, + image: str = "unclecode/crawl4ai:latest", + **kwargs + ) -> Dict: + """Start Crawl4AI server with specified configuration. + + Args: + replicas: Number of container replicas (default: 1) + mode: Deployment mode - 'auto', 'single', 'swarm', or 'compose' + port: External port to expose (default: 11235) + env_file: Path to environment file + image: Docker image to use + **kwargs: Additional docker run arguments + + Returns: + Dict with status and deployment info + """ + # Check if already running + state = self._load_state() + if state: + return { + "success": False, + "message": "Server already running", + "current_state": state + } + + # Validate Docker is available + if not self._is_docker_available(): + return { + "success": False, + "error": "Docker daemon not running. Please start Docker first." + } + + # Check port availability + if not self._is_port_available(port): + return { + "success": False, + "error": f"Port {port} is already in use" + } + + # Detect deployment mode + detected_mode = self._detect_mode(replicas, mode) + + # Ensure image is available + if not self._ensure_image(image): + return { + "success": False, + "error": f"Failed to pull image {image}" + } + + # Start based on mode + if detected_mode == "single": + result = self._start_single(port, env_file, image, **kwargs) + elif detected_mode == "swarm": + result = self._start_swarm(replicas, port, env_file, image, **kwargs) + elif detected_mode == "compose": + result = self._start_compose(replicas, port, env_file, image, **kwargs) + else: + return { + "success": False, + "error": f"Unknown mode: {detected_mode}" + } + + if result["success"]: + # Save state + self._save_state({ + "mode": detected_mode, + "replicas": replicas, + "port": port, + "image": image, + "env_file": env_file, + "started_at": datetime.now().isoformat(), + **result.get("state_data", {}) + }) + + return result + + async def status(self) -> Dict: + """Get current server status.""" + state = self._load_state() + + if not state: + return { + "running": False, + "message": "No server is currently running" + } + + mode = state["mode"] + + # Check actual container status + if mode == "single": + running = self._check_container_running(state.get("container_id")) + elif mode == "swarm": + running = self._check_service_running(state.get("service_name")) + elif mode == "compose": + running = self._check_compose_running(state.get("compose_project")) + else: + running = False + + if not running: + # State file exists but containers are gone - clean up + self._clear_state() + return { + "running": False, + "message": "State file exists but containers stopped externally" + } + + return { + "running": True, + "mode": mode, + "replicas": state.get("replicas", 1), + "port": state.get("port", 11235), + "image": state.get("image"), + "started_at": state.get("started_at"), + "uptime": self._calculate_uptime(state.get("started_at")) + } + + async def stop(self, remove_volumes: bool = False) -> Dict: + """Stop running server. + + Args: + remove_volumes: Remove associated volumes + + Returns: + Dict with stop status + """ + state = self._load_state() + + if not state: + return { + "success": False, + "message": "No server is running" + } + + mode = state["mode"] + + try: + if mode == "single": + self._stop_single(state.get("container_id"), remove_volumes) + elif mode == "swarm": + self._stop_swarm(state.get("service_name")) + elif mode == "compose": + self._stop_compose(state.get("compose_project"), remove_volumes) + + self._clear_state() + + return { + "success": True, + "message": f"Server stopped ({mode} mode)" + } + except Exception as e: + return { + "success": False, + "error": str(e) + } + + async def cleanup(self, force: bool = False) -> Dict: + """Force cleanup of all Crawl4AI Docker resources. + + Args: + force: Force cleanup even if state file doesn't exist + + Returns: + Dict with cleanup status + """ + import logging + logger = logging.getLogger(__name__) + + removed_count = 0 + messages = [] + + try: + # Try to stop via state file first + if not force: + state = self._load_state() + if state: + stop_result = await self.stop(remove_volumes=True) + if stop_result["success"]: + return { + "success": True, + "removed": 1, + "message": "Stopped via state file" + } + + # Force cleanup - find and remove all Crawl4AI resources + logger.info("Force cleanup: removing all Crawl4AI Docker resources") + + # Remove all crawl4ai containers + try: + result = subprocess.run( + ["docker", "ps", "-a", "--filter", "name=crawl4ai", "--format", "{{.ID}}"], + capture_output=True, + text=True, + timeout=10 + ) + container_ids = result.stdout.strip().split('\n') + container_ids = [cid for cid in container_ids if cid] + + for cid in container_ids: + subprocess.run(["docker", "rm", "-f", cid], capture_output=True, timeout=10) + removed_count += 1 + messages.append(f"Removed {len(container_ids)} crawl4ai containers") + except Exception as e: + logger.warning(f"Error removing containers: {e}") + + # Remove nginx containers + try: + result = subprocess.run( + ["docker", "ps", "-a", "--filter", "name=nginx", "--format", "{{.ID}}"], + capture_output=True, + text=True, + timeout=10 + ) + nginx_ids = result.stdout.strip().split('\n') + nginx_ids = [nid for nid in nginx_ids if nid] + + for nid in nginx_ids: + subprocess.run(["docker", "rm", "-f", nid], capture_output=True, timeout=10) + removed_count += len(nginx_ids) + if nginx_ids: + messages.append(f"Removed {len(nginx_ids)} nginx containers") + except Exception as e: + logger.warning(f"Error removing nginx: {e}") + + # Remove redis containers + try: + result = subprocess.run( + ["docker", "ps", "-a", "--filter", "name=redis", "--format", "{{.ID}}"], + capture_output=True, + text=True, + timeout=10 + ) + redis_ids = result.stdout.strip().split('\n') + redis_ids = [rid for rid in redis_ids if rid] + + for rid in redis_ids: + subprocess.run(["docker", "rm", "-f", rid], capture_output=True, timeout=10) + removed_count += len(redis_ids) + if redis_ids: + messages.append(f"Removed {len(redis_ids)} redis containers") + except Exception as e: + logger.warning(f"Error removing redis: {e}") + + # Clean up compose projects + for project in ["crawl4ai", "fix-docker"]: + try: + subprocess.run( + ["docker", "compose", "-p", project, "down", "-v"], + capture_output=True, + timeout=30, + cwd=str(self.state_dir) + ) + messages.append(f"Cleaned compose project: {project}") + except Exception: + pass + + # Remove networks + try: + subprocess.run(["docker", "network", "prune", "-f"], capture_output=True, timeout=10) + messages.append("Pruned networks") + except Exception as e: + logger.warning(f"Error pruning networks: {e}") + + # Clear state file + self._clear_state() + messages.append("Cleared state file") + + return { + "success": True, + "removed": removed_count, + "message": "; ".join(messages) + } + + except Exception as e: + logger.error(f"Cleanup error: {e}") + return { + "success": False, + "message": f"Cleanup failed: {str(e)}" + } + + async def scale(self, replicas: int) -> Dict: + """Scale server to specified replica count. + + Args: + replicas: Target number of replicas + + Returns: + Dict with scaling status + """ + state = self._load_state() + + if not state: + return { + "success": False, + "message": "No server is running" + } + + mode = state["mode"] + + if mode == "single": + return { + "success": False, + "error": "Cannot scale single container mode. Use 'crwl server stop' then 'crwl server start --replicas N'" + } + + try: + if mode == "swarm": + self._scale_swarm(state["service_name"], replicas) + elif mode == "compose": + self._scale_compose(state["compose_project"], replicas) + + # Update state + state["replicas"] = replicas + self._save_state(state) + + return { + "success": True, + "message": f"Scaled to {replicas} replicas", + "mode": mode + } + except Exception as e: + return { + "success": False, + "error": str(e) + } + + async def logs(self, follow: bool = False, tail: int = 100) -> str: + """Get server logs. + + Args: + follow: Follow log output + tail: Number of lines to show + + Returns: + Log output as string + """ + state = self._load_state() + + if not state: + return "No server is running" + + mode = state["mode"] + + try: + if mode == "single": + return self._logs_single(state["container_id"], follow, tail) + elif mode == "swarm": + return self._logs_swarm(state["service_name"], follow, tail) + elif mode == "compose": + return self._logs_compose(state["compose_project"], follow, tail) + except Exception as e: + return f"Error getting logs: {e}" + + # ========== Mode Detection ========== + + def _detect_mode(self, replicas: int, mode: str) -> ServerMode: + """Detect deployment mode based on replicas and user preference.""" + if mode != "auto": + return mode + + if replicas == 1: + return "single" + + # N>1: prefer Swarm if available, fallback to Compose + if self._is_swarm_available(): + return "swarm" + + return "compose" + + def _is_swarm_available(self) -> bool: + """Check if Docker Swarm is initialized and available.""" + try: + result = subprocess.run( + ["docker", "info", "--format", "{{.Swarm.LocalNodeState}}"], + capture_output=True, + text=True, + timeout=5 + ) + return result.stdout.strip() == "active" + except Exception: + return False + + def _is_docker_available(self) -> bool: + """Check if Docker daemon is running.""" + try: + subprocess.run( + ["docker", "ps"], + capture_output=True, + timeout=5, + check=True + ) + return True + except Exception: + return False + + def _is_port_available(self, port: int) -> bool: + """Check if port is available for binding.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(('0.0.0.0', port)) + return True + except OSError: + return False + + def _ensure_image(self, image: str) -> bool: + """Ensure Docker image is available locally, pull if needed.""" + try: + # Check if image exists locally + result = subprocess.run( + ["docker", "image", "inspect", image], + capture_output=True, + timeout=5 + ) + + if result.returncode == 0: + return True + + # Determine if this looks like a registry image + # Registry images have format: [registry/][namespace/]repository[:tag] + # Examples: unclecode/crawl4ai:latest, docker.io/library/nginx:latest + # Local-only: crawl4ai-local:latest, my-image:v1 + + # If it has a dot in the first part (before any slash), it's likely a registry + # Or if it has a slash, it's likely registry/namespace/repo format + parts = image.split("/") + is_registry_image = ( + len(parts) > 1 and # Has slash + "." not in parts[0] and # First part isn't a domain (localhost.localdomain) + not parts[0].startswith("localhost") # Not localhost registry + ) + + if not is_registry_image: + return False # Local image doesn't exist + + # Try to pull from registry + subprocess.run( + ["docker", "pull", image], + capture_output=True, + check=True, + timeout=300 + ) + return True + except Exception: + return False + + # ========== Single Container Mode ========== + + def _start_single(self, port: int, env_file: Optional[str], image: str, **kwargs) -> Dict: + """Start single container with docker run.""" + # Validate inputs to prevent injection attacks + if not validate_port(port): + return { + "success": False, + "error": f"Invalid port number: {port}. Must be between 1-65535." + } + + if not validate_docker_image(image): + return { + "success": False, + "error": f"Invalid Docker image format: {image}" + } + + if env_file and not validate_env_file(env_file): + return { + "success": False, + "error": f"Environment file not found or not readable: {env_file}" + } + + cmd = [ + "docker", "run", + "-d", # Detached + "--name", "crawl4ai_server", + "-p", f"{port}:11235", + "--shm-size=1g", # Important for browser + ] + + if env_file: + # Use absolute path to prevent path traversal + abs_env_file = str(Path(env_file).resolve()) + cmd.extend(["--env-file", abs_env_file]) + + # Whitelist allowed Docker flags to prevent privilege escalation + allowed_flags = {"--memory", "--cpus", "--restart", "--network"} + for key, value in kwargs.items(): + if key in allowed_flags: + cmd.append(key) + if value is not True: # Handle boolean flags + cmd.append(str(value)) + else: + # Log ignored flags for debugging + import logging + logging.warning(f"Ignoring non-whitelisted Docker flag: {key}") + + cmd.append(image) + + try: + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + container_id = result.stdout.strip() + + # Wait for health check + if self._wait_for_health(f"http://localhost:{port}/health"): + return { + "success": True, + "message": f"Server started on port {port}", + "state_data": {"container_id": container_id} + } + else: + # Cleanup failed container + subprocess.run(["docker", "rm", "-f", container_id], capture_output=True) + return { + "success": False, + "error": "Container started but health check failed" + } + except subprocess.CalledProcessError as e: + return { + "success": False, + "error": f"Failed to start container: {e.stderr}" + } + + def _stop_single(self, container_id: str, remove_volumes: bool): + """Stop single container.""" + cmd = ["docker", "rm", "-f"] + if remove_volumes: + cmd.append("-v") + cmd.append(container_id) + subprocess.run(cmd, check=True) + + def _check_container_running(self, container_id: str) -> bool: + """Check if container is running.""" + if not container_id: + return False + try: + result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", container_id], + capture_output=True, + text=True, + timeout=5 + ) + return result.stdout.strip() == "true" + except Exception: + return False + + def _logs_single(self, container_id: str, follow: bool, tail: int) -> str: + """Get logs from single container.""" + cmd = ["docker", "logs", "--tail", str(tail)] + if follow: + cmd.append("-f") + cmd.append(container_id) + + result = subprocess.run(cmd, capture_output=True, text=True) + return result.stdout + + # ========== Swarm Mode ========== + + def _start_swarm(self, replicas: int, port: int, env_file: Optional[str], image: str, **kwargs) -> Dict: + """Start service in Swarm mode.""" + # Validate inputs to prevent injection attacks + if not validate_replicas(replicas): + return { + "success": False, + "error": f"Invalid replica count: {replicas}. Must be between 1-100." + } + + if not validate_port(port): + return { + "success": False, + "error": f"Invalid port number: {port}. Must be between 1-65535." + } + + if not validate_docker_image(image): + return { + "success": False, + "error": f"Invalid Docker image format: {image}" + } + + if env_file and not validate_env_file(env_file): + return { + "success": False, + "error": f"Environment file not found or not readable: {env_file}" + } + + service_name = "crawl4ai" # Static name (safe) + + # Initialize swarm if needed + if not self._is_swarm_available(): + init_result = self._init_swarm() + if not init_result: + return { + "success": False, + "error": "Failed to initialize Docker Swarm. Use 'docker swarm init' manually." + } + + cmd = [ + "docker", "service", "create", + "--name", service_name, + "--replicas", str(replicas), + "--publish", f"{port}:11235", + "--mount", "type=tmpfs,target=/dev/shm,tmpfs-size=1g", + "--limit-memory", "4G", + ] + + if env_file: + # Use absolute path to prevent path traversal + abs_env_file = str(Path(env_file).resolve()) + cmd.extend(["--env-file", abs_env_file]) + + cmd.append(image) + + try: + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + service_id = result.stdout.strip() + + # Wait for service to be ready (check replicas) + if self._wait_for_service(service_name, replicas): + return { + "success": True, + "message": f"Swarm service started with {replicas} replicas", + "state_data": { + "service_name": service_name, + "service_id": service_id + } + } + else: + # Cleanup failed service + subprocess.run(["docker", "service", "rm", service_name], capture_output=True) + return { + "success": False, + "error": "Service created but replicas failed to start" + } + except subprocess.CalledProcessError as e: + return { + "success": False, + "error": f"Failed to create Swarm service: {e.stderr}" + } + + def _init_swarm(self) -> bool: + """Initialize Docker Swarm if not already initialized.""" + try: + result = subprocess.run( + ["docker", "swarm", "init"], + capture_output=True, + text=True, + timeout=10 + ) + return result.returncode == 0 + except Exception: + return False + + def _wait_for_service(self, service_name: str, expected_replicas: int, timeout: int = 60) -> bool: + """Wait for Swarm service replicas to be running.""" + import time + start = time.time() + + while time.time() - start < timeout: + try: + result = subprocess.run( + ["docker", "service", "ls", "--filter", f"name={service_name}", "--format", "{{.Replicas}}"], + capture_output=True, + text=True, + timeout=5 + ) + + if result.returncode == 0: + # Format is "2/3" (running/desired) + replicas_str = result.stdout.strip() + if "/" in replicas_str: + running, desired = replicas_str.split("/") + if int(running) == expected_replicas and int(desired) == expected_replicas: + return True + + time.sleep(2) + except Exception: + time.sleep(2) + + return False + + def _stop_swarm(self, service_name: str): + """Stop Swarm service.""" + subprocess.run( + ["docker", "service", "rm", service_name], + check=True, + capture_output=True + ) + + def _scale_swarm(self, service_name: str, replicas: int): + """Scale Swarm service.""" + subprocess.run( + ["docker", "service", "scale", f"{service_name}={replicas}"], + check=True, + capture_output=True + ) + + def _check_service_running(self, service_name: str) -> bool: + """Check if Swarm service is running.""" + if not service_name: + return False + try: + result = subprocess.run( + ["docker", "service", "ls", "--filter", f"name={service_name}", "--format", "{{.Name}}"], + capture_output=True, + text=True, + timeout=5 + ) + return service_name in result.stdout + except Exception: + return False + + def _logs_swarm(self, service_name: str, follow: bool, tail: int) -> str: + """Get logs from Swarm service.""" + cmd = ["docker", "service", "logs", "--tail", str(tail)] + if follow: + cmd.append("-f") + cmd.append(service_name) + + result = subprocess.run(cmd, capture_output=True, text=True) + return result.stdout + + # ========== Compose Mode ========== + + def _start_compose(self, replicas: int, port: int, env_file: Optional[str], image: str, **kwargs) -> Dict: + """Start with Docker Compose + Nginx.""" + # Validate inputs to prevent injection attacks + if not validate_replicas(replicas): + return { + "success": False, + "error": f"Invalid replica count: {replicas}. Must be between 1-100." + } + + if not validate_port(port): + return { + "success": False, + "error": f"Invalid port number: {port}. Must be between 1-65535." + } + + if not validate_docker_image(image): + return { + "success": False, + "error": f"Invalid Docker image format: {image}" + } + + if env_file and not validate_env_file(env_file): + return { + "success": False, + "error": f"Environment file not found or not readable: {env_file}" + } + + project_name = "crawl4ai" # Static name (safe) + + # Generate compose and nginx config files + try: + self._generate_compose_file(replicas, port, env_file or "", image) + self._generate_nginx_config() + except Exception as e: + return { + "success": False, + "error": f"Failed to generate config files: {e}" + } + + # Start compose stack - use absolute path for compose file + cmd = [ + "docker", "compose", + "-f", str(self.compose_file.resolve()), + "-p", project_name, + "up", "-d", + "--scale", f"crawl4ai={replicas}" + ] + + try: + result = subprocess.run(cmd, capture_output=True, text=True, check=True, cwd=str(self.state_dir)) + + # Wait for services to be healthy + if self._wait_for_compose_healthy(project_name, timeout=60): + return { + "success": True, + "message": f"Compose stack started with {replicas} replicas", + "state_data": { + "compose_project": project_name + } + } + else: + # Cleanup failed deployment + subprocess.run( + ["docker", "compose", "-f", str(self.compose_file), "-p", project_name, "down"], + capture_output=True, + cwd=str(self.state_dir) + ) + return { + "success": False, + "error": "Compose stack started but health checks failed" + } + except subprocess.CalledProcessError as e: + return { + "success": False, + "error": f"Failed to start Compose stack: {e.stderr}" + } + + def _generate_compose_file(self, replicas: int, port: int, env_file: str, image: str): + """Generate docker-compose.yml from template with validation.""" + import os + + # Get template path - check if we're in the package or dev environment + template_path = Path(__file__).parent / "templates" / "docker-compose.template.yml" + + if not template_path.exists(): + raise FileNotFoundError( + f"Docker Compose template not found: {template_path}\n" + f"Please ensure crawl4ai package is correctly installed.\n" + f"Try: pip install --force-reinstall crawl4ai" + ) + + try: + with open(template_path) as f: + template = f.read() + except IOError as e: + raise RuntimeError(f"Failed to read template {template_path}: {e}") + + # Validate template has required placeholders + required_vars = {"${IMAGE}", "${REPLICAS}", "${PORT}", "${NGINX_CONF}"} + missing = required_vars - set(re.findall(r'\$\{[A-Z_]+\}', template)) + if missing: + raise ValueError(f"Template missing required variables: {missing}") + + # Substitute variables + content = template.replace("${IMAGE}", image) + content = content.replace("${REPLICAS}", str(replicas)) + content = content.replace("${PORT}", str(port)) + content = content.replace("${NGINX_CONF}", str(self.nginx_conf)) + + # Verify no unsubstituted variables remain + remaining = re.findall(r'\$\{[A-Z_]+\}', content) + if remaining: + import logging + logging.warning(f"Unsubstituted variables in template: {remaining}") + + try: + with open(self.compose_file, "w") as f: + f.write(content) + except IOError as e: + raise RuntimeError(f"Failed to write compose file {self.compose_file}: {e}") + + def _generate_nginx_config(self): + """Generate nginx.conf from template with validation.""" + template_path = Path(__file__).parent / "templates" / "nginx.conf.template" + + if not template_path.exists(): + raise FileNotFoundError( + f"Nginx template not found: {template_path}\n" + f"Please ensure crawl4ai package is correctly installed.\n" + f"Try: pip install --force-reinstall crawl4ai" + ) + + try: + with open(template_path) as f: + content = f.read() + except IOError as e: + raise RuntimeError(f"Failed to read nginx template {template_path}: {e}") + + # Nginx template doesn't need variable substitution currently + try: + with open(self.nginx_conf, "w") as f: + f.write(content) + except IOError as e: + raise RuntimeError(f"Failed to write nginx config {self.nginx_conf}: {e}") + + def _wait_for_compose_healthy(self, project: str, timeout: int = 60) -> bool: + """Wait for Compose services to be healthy.""" + import time + start = time.time() + + while time.time() - start < timeout: + try: + # Check if nginx service is running (it depends on crawl4ai) + result = subprocess.run( + ["docker", "compose", "-f", str(self.compose_file), "-p", project, "ps", "--format", "json"], + capture_output=True, + text=True, + timeout=5, + cwd=str(self.state_dir) + ) + + if result.returncode == 0 and result.stdout: + import json + services = [json.loads(line) for line in result.stdout.strip().split('\n') if line] + + # Check if nginx is running (implies crawl4ai instances are up) + nginx_running = any( + s.get("Service") == "nginx" and s.get("State") == "running" + for s in services + ) + + if nginx_running: + return True + + time.sleep(2) + except Exception: + time.sleep(2) + + return False + + def _stop_compose(self, project: str, remove_volumes: bool): + """Stop Compose stack.""" + cmd = ["docker", "compose", "-f", str(self.compose_file), "-p", project, "down"] + if remove_volumes: + cmd.append("-v") + + subprocess.run(cmd, check=True, capture_output=True, cwd=str(self.state_dir)) + + def _scale_compose(self, project: str, replicas: int): + """Scale Compose service.""" + subprocess.run( + ["docker", "compose", "-f", str(self.compose_file), "-p", project, "up", "-d", "--scale", f"crawl4ai={replicas}", "--no-recreate"], + check=True, + capture_output=True, + cwd=str(self.state_dir) + ) + + def _check_compose_running(self, project: str) -> bool: + """Check if Compose stack is running.""" + if not project or not self.compose_file.exists(): + return False + try: + result = subprocess.run( + ["docker", "compose", "-f", str(self.compose_file), "-p", project, "ps", "-q"], + capture_output=True, + text=True, + timeout=5, + cwd=str(self.state_dir) + ) + # If there are any container IDs, the stack is running + return bool(result.stdout.strip()) + except Exception: + return False + + def _logs_compose(self, project: str, follow: bool, tail: int) -> str: + """Get logs from Compose stack.""" + cmd = ["docker", "compose", "-f", str(self.compose_file), "-p", project, "logs", "--tail", str(tail)] + if follow: + cmd.append("-f") + + result = subprocess.run(cmd, capture_output=True, text=True, cwd=str(self.state_dir)) + return result.stdout + + # ========== State Management ========== + + def _save_state(self, state: Dict): + """Persist server state to disk with atomic write and file locking.""" + import fcntl + + self.state_dir.mkdir(parents=True, exist_ok=True) + + # Atomic write with exclusive lock + temp_file = self.state_file.with_suffix('.tmp') + try: + with open(temp_file, 'w') as f: + fcntl.flock(f.fileno(), fcntl.LOCK_EX) # Exclusive lock + json.dump(state, f, indent=2) + f.flush() + os.fsync(f.fileno()) # Force write to disk + fcntl.flock(f.fileno(), fcntl.LOCK_UN) # Unlock + + # Atomic rename + temp_file.replace(self.state_file) + except Exception as e: + # Cleanup temp file on error + temp_file.unlink(missing_ok=True) + raise RuntimeError(f"Failed to save state: {e}") + + def _load_state(self) -> Optional[Dict]: + """Load server state from disk with file locking.""" + import fcntl + + if not self.state_file.exists(): + return None + + try: + with open(self.state_file) as f: + fcntl.flock(f.fileno(), fcntl.LOCK_SH) # Shared lock (read) + state = json.load(f) + fcntl.flock(f.fileno(), fcntl.LOCK_UN) # Unlock + return state + except (json.JSONDecodeError, IOError) as e: + # Log and remove corrupted state file + import logging + logging.error(f"Corrupted state file, removing: {e}") + self.state_file.unlink(missing_ok=True) + return None + + def _clear_state(self): + """Remove state file with locking.""" + import fcntl + + if self.state_file.exists(): + try: + # Acquire lock before deletion to prevent race + with open(self.state_file, 'r') as f: + fcntl.flock(f.fileno(), fcntl.LOCK_EX) + # Lock acquired, now delete + self.state_file.unlink(missing_ok=True) + except Exception: + # If lock fails, force delete anyway + self.state_file.unlink(missing_ok=True) + + # ========== Helpers ========== + + def _wait_for_health(self, url: str, timeout: int = 30) -> bool: + """Wait for health endpoint to respond.""" + import urllib.request + + start = time.time() + while time.time() - start < timeout: + try: + urllib.request.urlopen(url, timeout=2) + return True + except Exception: + time.sleep(1) + return False + + def _calculate_uptime(self, started_at: str) -> str: + """Calculate uptime from ISO timestamp.""" + if not started_at: + return "unknown" + + try: + start = datetime.fromisoformat(started_at) + delta = datetime.now() - start + + hours = delta.seconds // 3600 + minutes = (delta.seconds % 3600) // 60 + + if delta.days > 0: + return f"{delta.days}d {hours}h {minutes}m" + elif hours > 0: + return f"{hours}h {minutes}m" + else: + return f"{minutes}m" + except Exception: + return "unknown" diff --git a/deploy/installer/deploy.sh b/deploy/installer/deploy.sh new file mode 100755 index 00000000..95b984ed --- /dev/null +++ b/deploy/installer/deploy.sh @@ -0,0 +1,217 @@ +#!/bin/bash +# Crawl4AI Node Manager (cnode) Installation Script +# Usage: curl -sSL https://crawl4ai.com/deploy.sh | bash +# Or: wget -qO- https://crawl4ai.com/deploy.sh | bash + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +INSTALL_DIR="${INSTALL_DIR:-/usr/local/bin}" +BINARY_NAME="cnode" +GITHUB_REPO="unclecode/crawl4ai" +RELEASE_TAG="${CNODE_VERSION:-latest}" + +echo -e "${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}" +echo -e "${GREEN}║ Crawl4AI Node Manager (cnode) Installation Script ║${NC}" +echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}\n" + +# Detect OS and architecture +detect_platform() { + OS="$(uname -s)" + ARCH="$(uname -m)" + + case "$OS" in + Linux*) + OS_TYPE="linux" + ;; + Darwin*) + OS_TYPE="macos" + ;; + *) + echo -e "${RED}Error: Unsupported operating system: $OS${NC}" + exit 1 + ;; + esac + + case "$ARCH" in + x86_64|amd64) + ARCH_TYPE="amd64" + ;; + aarch64|arm64) + ARCH_TYPE="arm64" + ;; + *) + echo -e "${RED}Error: Unsupported architecture: $ARCH${NC}" + exit 1 + ;; + esac + + echo -e "${BLUE}Detected platform: ${YELLOW}$OS_TYPE-$ARCH_TYPE${NC}" +} + +# Check if Docker is installed +check_docker() { + if ! command -v docker &> /dev/null; then + echo -e "${YELLOW}⚠️ Docker not found. cnode requires Docker to manage server instances.${NC}" + echo -e "${YELLOW}Install Docker from: https://docs.docker.com/get-docker/${NC}\n" + read -p "Continue installation anyway? (y/N) " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi + else + echo -e "${GREEN}✓ Docker is installed${NC}" + fi +} + +# Check write permissions +check_permissions() { + if [ ! -w "$INSTALL_DIR" ]; then + echo -e "${YELLOW}⚠️ No write permission for $INSTALL_DIR${NC}" + echo -e "${YELLOW}The script will attempt to use sudo for installation.${NC}\n" + USE_SUDO="sudo" + else + USE_SUDO="" + fi +} + +# Download binary +download_binary() { + BINARY_URL="https://github.com/$GITHUB_REPO/releases/download/$RELEASE_TAG/cnode-$OS_TYPE-$ARCH_TYPE" + + echo -e "${BLUE}Downloading cnode from GitHub...${NC}" + echo -e "${YELLOW}URL: $BINARY_URL${NC}\n" + + # Create temp directory + TMP_DIR="$(mktemp -d)" + TMP_FILE="$TMP_DIR/$BINARY_NAME" + + # Download with curl or wget + if command -v curl &> /dev/null; then + if ! curl -fSL "$BINARY_URL" -o "$TMP_FILE"; then + echo -e "${RED}Error: Failed to download binary${NC}" + echo -e "${YELLOW}URL: $BINARY_URL${NC}" + rm -rf "$TMP_DIR" + exit 1 + fi + elif command -v wget &> /dev/null; then + if ! wget -q "$BINARY_URL" -O "$TMP_FILE"; then + echo -e "${RED}Error: Failed to download binary${NC}" + echo -e "${YELLOW}URL: $BINARY_URL${NC}" + rm -rf "$TMP_DIR" + exit 1 + fi + else + echo -e "${RED}Error: Neither curl nor wget found${NC}" + echo -e "${YELLOW}Please install curl or wget and try again${NC}" + rm -rf "$TMP_DIR" + exit 1 + fi + + # Make executable + chmod +x "$TMP_FILE" + + echo "$TMP_FILE" +} + +# Install binary +install_binary() { + local tmp_file="$1" + local install_path="$INSTALL_DIR/$BINARY_NAME" + + echo -e "\n${BLUE}Installing cnode to $install_path...${NC}" + + # Check if already installed + if [ -f "$install_path" ]; then + echo -e "${YELLOW}⚠️ cnode is already installed${NC}" + read -p "Overwrite existing installation? (y/N) " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo -e "${YELLOW}Installation cancelled${NC}" + rm -rf "$(dirname "$tmp_file")" + exit 0 + fi + fi + + # Install + if ! $USE_SUDO mv "$tmp_file" "$install_path"; then + echo -e "${RED}Error: Failed to install binary${NC}" + rm -rf "$(dirname "$tmp_file")" + exit 1 + fi + + # Cleanup temp directory + rm -rf "$(dirname "$tmp_file")" + + echo -e "${GREEN}✓ Installation successful${NC}" +} + +# Verify installation +verify_installation() { + echo -e "\n${BLUE}Verifying installation...${NC}" + + if ! command -v $BINARY_NAME &> /dev/null; then + echo -e "${RED}Error: $BINARY_NAME not found in PATH${NC}" + echo -e "${YELLOW}You may need to add $INSTALL_DIR to your PATH${NC}" + echo -e "${YELLOW}Add this to your ~/.bashrc or ~/.zshrc:${NC}" + echo -e "${YELLOW}export PATH=\"$INSTALL_DIR:\$PATH\"${NC}\n" + exit 1 + fi + + # Test version + if $BINARY_NAME --help &> /dev/null; then + echo -e "${GREEN}✓ $BINARY_NAME is working correctly${NC}" + else + echo -e "${RED}Error: $BINARY_NAME failed to execute${NC}" + exit 1 + fi +} + +# Show completion message +show_completion() { + local version + version=$($BINARY_NAME --help | head -1 || echo "unknown") + + echo -e "\n${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}" + echo -e "${GREEN}║ Installation Complete! ║${NC}" + echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}\n" + + echo -e "${BLUE}cnode is now installed and ready to use!${NC}\n" + + echo -e "${YELLOW}Quick Start:${NC}" + echo -e " ${GREEN}cnode start${NC} # Start single server instance" + echo -e " ${GREEN}cnode start --replicas 5${NC} # Start 5-replica cluster" + echo -e " ${GREEN}cnode status${NC} # Check server status" + echo -e " ${GREEN}cnode logs -f${NC} # Follow server logs" + echo -e " ${GREEN}cnode scale 10${NC} # Scale to 10 replicas" + echo -e " ${GREEN}cnode stop${NC} # Stop server" + + echo -e "\n${YELLOW}For more information:${NC}" + echo -e " ${BLUE}cnode --help${NC}" + echo -e " ${BLUE}https://github.com/$GITHUB_REPO${NC}\n" +} + +# Main installation flow +main() { + detect_platform + check_docker + check_permissions + + # Download and install + TMP_FILE=$(download_binary) + install_binary "$TMP_FILE" + + # Verify + verify_installation + show_completion +} + +# Run installation +main diff --git a/deploy/installer/install-cnode.sh b/deploy/installer/install-cnode.sh new file mode 100755 index 00000000..11c9bb1a --- /dev/null +++ b/deploy/installer/install-cnode.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# Crawl4AI Node Manager (cnode) - Fast Installation Script +# This installs cnode as a Python package with a wrapper script + +set -e + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +echo -e "${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}" +echo -e "${GREEN}║ Crawl4AI Node Manager (cnode) - Fast Installer ║${NC}" +echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}\n" + +# Configuration +INSTALL_DIR="${INSTALL_DIR:-/usr/local/bin}" +LIB_DIR="${LIB_DIR:-/usr/local/lib/cnode}" + +# Check Python +echo -e "${BLUE}Checking Python installation...${NC}" +if command -v python3 &> /dev/null; then + PYTHON_CMD="python3" +elif command -v python &> /dev/null; then + PYTHON_CMD="python" +else + echo -e "${RED}Error: Python 3.8+ is required but not found${NC}" + echo -e "${YELLOW}Install from: https://www.python.org/downloads/${NC}" + exit 1 +fi + +# Check Python version +PYTHON_VERSION=$($PYTHON_CMD -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') +echo -e "${GREEN}✓ Found Python $PYTHON_VERSION${NC}" + +if [ "$(printf '%s\n' "3.8" "$PYTHON_VERSION" | sort -V | head -n1)" != "3.8" ]; then + echo -e "${RED}Error: Python 3.8+ required, found $PYTHON_VERSION${NC}" + exit 1 +fi + +# Check pip +if ! $PYTHON_CMD -m pip --version &> /dev/null; then + echo -e "${RED}Error: pip is required${NC}" + echo -e "${YELLOW}Install pip: $PYTHON_CMD -m ensurepip${NC}" + exit 1 +fi + +echo -e "${GREEN}✓ pip is available${NC}" + +# Check Docker +echo -e "\n${BLUE}Checking Docker...${NC}" +if ! command -v docker &> /dev/null; then + echo -e "${YELLOW}⚠️ Docker not found (required for running servers)${NC}" + echo -e "${YELLOW}Install from: https://docs.docker.com/get-docker/${NC}\n" +else + echo -e "${GREEN}✓ Docker is installed${NC}" +fi + +# Check permissions +USE_SUDO="" +if [ ! -w "$INSTALL_DIR" ] || [ ! -w "/usr/local" ]; then + echo -e "\n${YELLOW}⚠️ Root permission required for installation${NC}" + USE_SUDO="sudo" +fi + +# Get script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Install Python dependencies +echo -e "\n${BLUE}Installing Python dependencies...${NC}" +$PYTHON_CMD -m pip install --quiet --user -r "$SCRIPT_DIR/cnode_pkg/requirements.txt" 2>/dev/null || \ +$PYTHON_CMD -m pip install --quiet --user --break-system-packages -r "$SCRIPT_DIR/cnode_pkg/requirements.txt" 2>/dev/null || { + echo -e "${YELLOW}⚠️ Could not install dependencies with pip${NC}" + echo -e "${YELLOW}Trying to continue anyway (dependencies may already be installed)${NC}" +} +echo -e "${GREEN}✓ Dependencies check complete${NC}" + +# Create lib directory +echo -e "\n${BLUE}Installing cnode package...${NC}" +$USE_SUDO mkdir -p "$LIB_DIR" +$USE_SUDO cp -r "$SCRIPT_DIR/cnode_pkg" "$LIB_DIR/" +echo -e "${GREEN}✓ Package installed to $LIB_DIR${NC}" + +# Create wrapper script +echo -e "\n${BLUE}Creating cnode command...${NC}" +$USE_SUDO tee "$INSTALL_DIR/cnode" > /dev/null << 'EOF' +#!/usr/bin/env bash +# Crawl4AI Node Manager (cnode) wrapper + +set -e + +# Find Python +if command -v python3 &> /dev/null; then + PYTHON_CMD="python3" +elif command -v python &> /dev/null; then + PYTHON_CMD="python" +else + echo "Error: Python 3.8+ required" >&2 + exit 1 +fi + +# Add cnode to Python path and run +export PYTHONPATH="/usr/local/lib/cnode:$PYTHONPATH" +exec $PYTHON_CMD -m cnode_pkg.cli "$@" +EOF + +$USE_SUDO chmod +x "$INSTALL_DIR/cnode" +echo -e "${GREEN}✓ cnode command created${NC}" + +# Verify installation +echo -e "\n${BLUE}Verifying installation...${NC}" +if ! command -v cnode &> /dev/null; then + echo -e "${RED}Error: cnode not found in PATH${NC}" + echo -e "${YELLOW}Add $INSTALL_DIR to your PATH${NC}" + exit 1 +fi + +if ! cnode --help &> /dev/null; then + echo -e "${RED}Error: cnode command failed${NC}" + exit 1 +fi + +echo -e "${GREEN}✓ Installation verified${NC}" + +# Success message +echo -e "\n${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}" +echo -e "${GREEN}║ Installation Complete! ║${NC}" +echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}\n" + +echo -e "${BLUE}cnode is now installed and ready!${NC}\n" + +echo -e "${YELLOW}Quick Start:${NC}" +echo -e " ${GREEN}cnode start${NC} # Start single server" +echo -e " ${GREEN}cnode start --replicas 5${NC} # Start 5-replica cluster" +echo -e " ${GREEN}cnode status${NC} # Check status" +echo -e " ${GREEN}cnode logs -f${NC} # Follow logs" +echo -e " ${GREEN}cnode stop${NC} # Stop server" + +echo -e "\n${YELLOW}More help:${NC}" +echo -e " ${BLUE}cnode --help${NC}\n" diff --git a/deploy/installer/sync-cnode.sh b/deploy/installer/sync-cnode.sh new file mode 100755 index 00000000..7eb88339 --- /dev/null +++ b/deploy/installer/sync-cnode.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Sync cnode source code to installer package +# Run this before committing changes to cnode + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOURCE_DIR="$SCRIPT_DIR/../docker" +PKG_DIR="$SCRIPT_DIR/cnode_pkg" + +echo "🔄 Syncing cnode source to package..." + +# Copy CLI +echo " → Copying cnode_cli.py to cli.py" +cp "$SOURCE_DIR/cnode_cli.py" "$PKG_DIR/cli.py" + +# Fix imports +echo " → Fixing imports (deploy.docker → cnode_pkg)" +sed -i '' 's/from deploy\.docker\./from cnode_pkg./g' "$PKG_DIR/cli.py" + +# Copy server manager +echo " → Copying server_manager.py" +cp "$SOURCE_DIR/server_manager.py" "$PKG_DIR/server_manager.py" + +echo "✅ Sync complete!" +echo "" +echo "Files updated:" +echo " • deploy/installer/cnode_pkg/cli.py" +echo " • deploy/installer/cnode_pkg/server_manager.py" +echo "" +echo "Next steps:" +echo " 1. Test: cd deploy/installer && ./install-cnode.sh" +echo " 2. Commit both source and package files" diff --git a/setup-hooks.sh b/setup-hooks.sh new file mode 100755 index 00000000..29a66d90 --- /dev/null +++ b/setup-hooks.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Setup Git hooks for cnode auto-sync +# Run this once after cloning the repo: ./setup-hooks.sh + +set -e + +echo "🔧 Setting up Git hooks..." + +# Configure Git to use .githooks directory +git config core.hooksPath .githooks + +echo "✅ Git hooks configured!" +echo "" +echo "Hooks installed:" +echo " • pre-commit: Auto-syncs cnode source → package when committing" +echo "" +echo "What this means:" +echo " ✅ Edit deploy/docker/cnode_cli.py" +echo " ✅ Run: git add deploy/docker/cnode_cli.py" +echo " ✅ Run: git commit -m \"update cnode\"" +echo " ✅ Hook automatically syncs to deploy/installer/cnode_pkg/" +echo " ✅ Synced files are auto-staged in the same commit" +echo "" +echo "You're all set! 🚀" diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index b7416dc2..c2c287b8 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -65,14 +65,14 @@ class TestCLIBasics: assert 'Crawl4AI CLI' in result.output def test_examples(self, runner): - result = runner.invoke(cli, ['--example']) + result = runner.invoke(cli, ['examples']) assert result.exit_code == 0 assert 'Examples' in result.output def test_missing_url(self, runner): - result = runner.invoke(cli) + result = runner.invoke(cli, ['crawl']) assert result.exit_code != 0 - assert 'URL argument is required' in result.output + assert ('Missing argument' in result.output or 'required' in result.output.lower()) class TestConfigParsing: def test_parse_key_values_basic(self): @@ -101,18 +101,19 @@ class TestConfigLoading: class TestLLMConfig: def test_llm_config_creation(self, temp_config_dir, runner): def input_simulation(inputs): - return runner.invoke(cli, ['https://example.com', '-q', 'test question'], + return runner.invoke(cli, ['crawl', 'https://example.com', '-q', 'test question'], input='\n'.join(inputs)) class TestCrawlingFeatures: def test_basic_crawl(self, runner): - result = runner.invoke(cli, ['https://example.com']) + result = runner.invoke(cli, ['crawl', 'https://example.com']) assert result.exit_code == 0 class TestErrorHandling: def test_invalid_config_file(self, runner): result = runner.invoke(cli, [ + 'crawl', 'https://example.com', '--browser-config', 'nonexistent.yml' ]) @@ -122,8 +123,9 @@ class TestErrorHandling: invalid_schema = temp_config_dir / 'invalid_schema.json' with open(invalid_schema, 'w') as f: f.write('invalid json') - + result = runner.invoke(cli, [ + 'crawl', 'https://example.com', '--schema', str(invalid_schema) ])