ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cnode): add standalone CLI for Docker server management

Browse Source 
- Reorganized server management code:
  - Moved server_cli.py -> deploy/docker/cnode_cli.py
  - Moved server_manager.py -> deploy/docker/server_manager.py

- Created fast Python-based installation (0.1s startup):
  - deploy/installer/cnode_pkg/ - Standalone package
  - deploy/installer/install-cnode.sh - Local installer
  - deploy/installer/deploy.sh - Remote installer for users

- Added backward compatibility:
  - crawl4ai/cli.py: 'crwl server' redirects to 'cnode'
  - Updated tests to match new CLI structure (12/12 passing)

- Automated sync workflow:
  - .githooks/pre-commit - Auto-syncs source to package
  - setup-hooks.sh - One-time setup for contributors
  - deploy/installer/sync-cnode.sh - Manual sync script

Performance:
  - Startup time: 0.1s (49x faster than PyInstaller)
  - Size: ~50KB wrapper vs 8.8MB binary

Commands:
  cnode start [--replicas N]  # Start server/cluster
  cnode status                # Check status
  cnode scale N               # Scale replicas
  cnode logs [-f]             # View logs
  cnode stop                  # Stop server
This commit is contained in:
unclecode
2025-10-21 09:31:18 +08:00
parent 342fc52b47
commit cd02616218
16 changed files with 4181 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
.githooks/pre-commit Executable file
View File

@@ -0,0 +1,31 @@
#!/bin/bash
# Pre-commit hook: Auto-sync cnode files when cnode source is modified
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'
# Check if cnode source files are being committed
CNODE_FILES_CHANGED=$(git diff --cached --name-only | grep -E "deploy/docker/(cnode_cli|server_manager)\.py")
if [ -n "$CNODE_FILES_CHANGED" ]; then
echo -e "${YELLOW}🔄 cnode source files modified, auto-syncing to package...${NC}"
# Run sync script
if [ -f "deploy/installer/sync-cnode.sh" ]; then
bash deploy/installer/sync-cnode.sh
# Stage the synced files
git add deploy/installer/cnode_pkg/cli.py
git add deploy/installer/cnode_pkg/server_manager.py
echo -e "${GREEN}✅ cnode package synced and staged${NC}"
else
echo -e "${RED}❌ Error: sync-cnode.sh not found${NC}"
exit 1
fi
fi
exit 0

3
.gitignore vendored
View File

@@ -185,7 +185,8 @@ Crawl4AI.egg-info/
requirements0.txt
a.txt
*.sh
# Ignore shell scripts globally, but allow test scripts
# *.sh
.idea
docs/examples/.chainlit/
docs/examples/.chainlit/*

81
crawl4ai/cli.py
View File

@@ -2,6 +2,8 @@ import click
import os
import sys
import time
import subprocess
import shutil
import humanize
from typing import Dict, Any, Optional, List
@@ -626,8 +628,73 @@ def cli():
# Register server command group (Docker orchestration)
from crawl4ai.server_cli import server_cmd
cli.add_command(server_cmd)
# Redirect to standalone 'cnode' CLI
@cli.command("server", context_settings=dict(
ignore_unknown_options=True,
allow_extra_args=True,
allow_interspersed_args=False
))
@click.pass_context
def server_cmd(ctx):
"""Manage Crawl4AI Docker server instances (deprecated - use 'cnode')
This command has been moved to a standalone CLI called 'cnode'.
For new installations, use:
curl -sSL https://crawl4ai.com/deploy.sh | bash
This redirect allows existing scripts to continue working.
Available commands: start, stop, status, scale, logs
Use 'crwl server <command> --help' for command-specific help.
"""
# Check if cnode is installed
cnode_path = shutil.which("cnode")
# Get all the args (subcommand + options)
args = ctx.args
if not cnode_path:
console.print(Panel(
"[yellow]The 'crwl server' command has been moved to a standalone CLI.[/yellow]\n\n"
"Please install 'cnode' (Crawl4AI Node Manager):\n"
"[cyan]curl -sSL https://crawl4ai.com/deploy.sh | bash[/cyan]\n\n"
"After installation, use:\n"
"[green]cnode <command>[/green] instead of [dim]crwl server <command>[/dim]\n\n"
"For backward compatibility, we're using the local version for now.",
title="Server Command Moved",
border_style="yellow"
))
# Try to use local version
try:
import sys
# Add deploy/docker to path
deploy_path = str(Path(__file__).parent.parent / 'deploy' / 'docker')
if deploy_path not in sys.path:
sys.path.insert(0, deploy_path)
from cnode_cli import cli as cnode_cli
# Forward to cnode with the args
sys.argv = ['cnode'] + args
cnode_cli(standalone_mode=False)
sys.exit(0)
except SystemExit as e:
# Normal exit from click
sys.exit(e.code if hasattr(e, 'code') else 0)
except Exception as e:
console.print(f"[red]Error: Could not find cnode or local server CLI: {e}[/red]")
console.print(f"[dim]Details: {e}[/dim]")
import traceback
console.print(f"[dim]{traceback.format_exc()}[/dim]")
sys.exit(1)
# cnode is installed - forward everything to it
try:
result = subprocess.run([cnode_path] + args, check=False)
sys.exit(result.returncode)
except Exception as e:
console.print(f"[red]Error running cnode: {e}[/red]")
sys.exit(1)
@cli.group("browser")
@@ -1467,9 +1534,15 @@ def default(url: str, example: bool, browser_config: str, crawler_config: str, f
def main():
import sys
if len(sys.argv) < 2 or sys.argv[1] not in cli.commands:
# Don't auto-insert 'crawl' if the command is recognized
if len(sys.argv) >= 2 and sys.argv[1] in cli.commands:
cli()
elif len(sys.argv) < 2:
cli()
else:
# Unknown command - insert 'crawl' for backward compat
sys.argv.insert(1, "crawl")
cli()
cli()
if __name__ == "__main__":
main()

1
deploy/docker/__init__.py Normal file
View File

@@ -0,0 +1 @@
# Deploy docker module

492
deploy/docker/cnode_cli.py Normal file
View File

@@ -0,0 +1,492 @@
"""
Crawl4AI Server CLI Commands
Provides `cnode` command group for Docker orchestration.
"""
import click
import anyio
from rich.console import Console
from rich.table import Table
from rich.panel import Panel
from rich.prompt import Confirm
from deploy.docker.server_manager import ServerManager
console = Console()
@click.group()
def cli():
"""Manage Crawl4AI Docker server instances
\b
One-command deployment with automatic scaling:
• Single container for development (N=1)
• Docker Swarm for production with built-in load balancing (N>1)
• Docker Compose + Nginx as fallback (N>1)
\b
Examples:
cnode start # Single container on port 11235
cnode start --replicas 3 # Auto-detect Swarm or Compose
cnode start -r 5 --port 8080 # 5 replicas on custom port
cnode status # Check current deployment
cnode scale 10 # Scale to 10 replicas
cnode stop # Stop and cleanup
"""
pass
@cli.command("start")
@click.option(
"--replicas", "-r",
type=int,
default=1,
help="Number of container replicas (default: 1)"
)
@click.option(
"--mode",
type=click.Choice(["auto", "single", "swarm", "compose"]),
default="auto",
help="Deployment mode (default: auto-detect)"
)
@click.option(
"--port", "-p",
type=int,
default=11235,
help="External port to expose (default: 11235)"
)
@click.option(
"--env-file",
type=click.Path(exists=True),
help="Path to environment file"
)
@click.option(
"--image",
default="unclecode/crawl4ai:latest",
help="Docker image to use (default: unclecode/crawl4ai:latest)"
)
def start_cmd(replicas: int, mode: str, port: int, env_file: str, image: str):
"""Start Crawl4AI server with automatic orchestration.
Deployment modes:
- auto: Automatically choose best mode (default)
- single: Single container (N=1 only)
- swarm: Docker Swarm with built-in load balancing
- compose: Docker Compose + Nginx reverse proxy
The server will:
1. Check if Docker is running
2. Validate port availability
3. Pull image if needed
4. Start container(s) with health checks
5. Save state for management
Examples:
# Development: single container
cnode start
# Production: 5 replicas with Swarm
cnode start --replicas 5
# Custom configuration
cnode start -r 3 --port 8080 --env-file .env.prod
"""
manager = ServerManager()
console.print(Panel(
f"[cyan]Starting Crawl4AI Server[/cyan]\n\n"
f"Replicas: [yellow]{replicas}[/yellow]\n"
f"Mode: [yellow]{mode}[/yellow]\n"
f"Port: [yellow]{port}[/yellow]\n"
f"Image: [yellow]{image}[/yellow]",
title="Server Start",
border_style="cyan"
))
with console.status("[cyan]Starting server..."):
async def _start():
return await manager.start(
replicas=replicas,
mode=mode,
port=port,
env_file=env_file,
image=image
)
result = anyio.run(_start)
if result["success"]:
console.print(Panel(
f"[green]✓ Server started successfully![/green]\n\n"
f"Mode: [cyan]{result.get('state_data', {}).get('mode', mode)}[/cyan]\n"
f"URL: [bold]http://localhost:{port}[/bold]\n"
f"Health: [bold]http://localhost:{port}/health[/bold]\n"
f"Monitor: [bold]http://localhost:{port}/monitor[/bold]",
title="Server Running",
border_style="green"
))
else:
error_msg = result.get("error", result.get("message", "Unknown error"))
console.print(Panel(
f"[red]✗ Failed to start server[/red]\n\n"
f"{error_msg}",
title="Error",
border_style="red"
))
if "already running" in error_msg.lower():
console.print("\n[yellow]Hint: Use 'cnode status' to check current deployment[/yellow]")
console.print("[yellow] Use 'cnode stop' to stop existing server[/yellow]")
@cli.command("status")
def status_cmd():
"""Show current server status and deployment info.
Displays:
- Running state (up/down)
- Deployment mode (single/swarm/compose)
- Number of replicas
- Port mapping
- Uptime
- Image version
Example:
cnode status
"""
manager = ServerManager()
async def _status():
return await manager.status()
result = anyio.run(_status)
if result["running"]:
table = Table(title="Crawl4AI Server Status", border_style="green")
table.add_column("Property", style="cyan")
table.add_column("Value", style="green")
table.add_row("Status", "🟢 Running")
table.add_row("Mode", result["mode"])
table.add_row("Replicas", str(result.get("replicas", 1)))
table.add_row("Port", str(result.get("port", 11235)))
table.add_row("Image", result.get("image", "unknown"))
table.add_row("Uptime", result.get("uptime", "unknown"))
table.add_row("Started", result.get("started_at", "unknown"))
console.print(table)
console.print(f"\n[green]✓ Server is healthy[/green]")
console.print(f"[dim]Access: http://localhost:{result.get('port', 11235)}[/dim]")
else:
console.print(Panel(
f"[yellow]No server is currently running[/yellow]\n\n"
f"Use 'cnode start' to launch a server",
title="Server Status",
border_style="yellow"
))
@cli.command("stop")
@click.option(
"--remove-volumes",
is_flag=True,
help="Remove associated volumes (WARNING: deletes data)"
)
def stop_cmd(remove_volumes: bool):
"""Stop running Crawl4AI server and cleanup resources.
This will:
1. Stop all running containers/services
2. Remove containers
3. Optionally remove volumes (--remove-volumes)
4. Clean up state files
WARNING: Use --remove-volumes with caution as it will delete
persistent data including Redis databases and logs.
Examples:
# Stop server, keep volumes
cnode stop
# Stop and remove all data
cnode stop --remove-volumes
"""
manager = ServerManager()
# Confirm if removing volumes
if remove_volumes:
if not Confirm.ask(
"[red]⚠️ This will delete all server data including Redis databases. Continue?[/red]"
):
console.print("[yellow]Cancelled[/yellow]")
return
with console.status("[cyan]Stopping server..."):
async def _stop():
return await manager.stop(remove_volumes=remove_volumes)
result = anyio.run(_stop)
if result["success"]:
console.print(Panel(
f"[green]✓ Server stopped successfully[/green]\n\n"
f"{result.get('message', 'All resources cleaned up')}",
title="Server Stopped",
border_style="green"
))
else:
console.print(Panel(
f"[red]✗ Error stopping server[/red]\n\n"
f"{result.get('error', result.get('message', 'Unknown error'))}",
title="Error",
border_style="red"
))
@cli.command("scale")
@click.argument("replicas", type=int)
def scale_cmd(replicas: int):
"""Scale server to specified number of replicas.
Only works with Swarm or Compose modes. Single container
mode cannot be scaled (must stop and restart with --replicas).
Scaling is live and does not require downtime. The load
balancer will automatically distribute traffic to new replicas.
Examples:
# Scale up to 10 replicas
cnode scale 10
# Scale down to 2 replicas
cnode scale 2
# Scale to 1 (minimum)
cnode scale 1
"""
if replicas < 1:
console.print("[red]Error: Replicas must be at least 1[/red]")
return
manager = ServerManager()
with console.status(f"[cyan]Scaling to {replicas} replicas..."):
async def _scale():
return await manager.scale(replicas=replicas)
result = anyio.run(_scale)
if result["success"]:
console.print(Panel(
f"[green]✓ Scaled successfully[/green]\n\n"
f"New replica count: [bold]{replicas}[/bold]\n"
f"Mode: [cyan]{result.get('mode')}[/cyan]",
title="Scaling Complete",
border_style="green"
))
else:
error_msg = result.get("error", result.get("message", "Unknown error"))
console.print(Panel(
f"[red]✗ Scaling failed[/red]\n\n"
f"{error_msg}",
title="Error",
border_style="red"
))
if "single container" in error_msg.lower():
console.print("\n[yellow]Hint: For single container mode:[/yellow]")
console.print("[yellow] 1. cnode stop[/yellow]")
console.print(f"[yellow] 2. cnode start --replicas {replicas}[/yellow]")
@cli.command("logs")
@click.option(
"--follow", "-f",
is_flag=True,
help="Follow log output (like tail -f)"
)
@click.option(
"--tail",
type=int,
default=100,
help="Number of lines to show (default: 100)"
)
def logs_cmd(follow: bool, tail: int):
"""View server logs.
Shows logs from running containers/services. Use --follow
to stream logs in real-time.
Examples:
# Show last 100 lines
cnode logs
# Show last 500 lines
cnode logs --tail 500
# Follow logs in real-time
cnode logs --follow
# Combine options
cnode logs -f --tail 50
"""
manager = ServerManager()
async def _logs():
return await manager.logs(follow=follow, tail=tail)
output = anyio.run(_logs)
console.print(output)
@cli.command("cleanup")
@click.option(
"--force",
is_flag=True,
help="Force cleanup even if state file doesn't exist"
)
def cleanup_cmd(force: bool):
"""Force cleanup of all Crawl4AI Docker resources.
Stops and removes all containers, networks, and optionally volumes.
Useful when server is stuck or state is corrupted.
Examples:
# Clean up everything
cnode cleanup
# Force cleanup (ignore state file)
cnode cleanup --force
"""
manager = ServerManager()
console.print(Panel(
f"[yellow]⚠️ Cleaning up Crawl4AI Docker resources[/yellow]\n\n"
f"This will stop and remove:\n"
f"- All Crawl4AI containers\n"
f"- Nginx load balancer\n"
f"- Redis instance\n"
f"- Docker networks\n"
f"- State files",
title="Cleanup",
border_style="yellow"
))
if not force and not Confirm.ask("[yellow]Continue with cleanup?[/yellow]"):
console.print("[yellow]Cancelled[/yellow]")
return
with console.status("[cyan]Cleaning up resources..."):
async def _cleanup():
return await manager.cleanup(force=force)
result = anyio.run(_cleanup)
if result["success"]:
console.print(Panel(
f"[green]✓ Cleanup completed successfully[/green]\n\n"
f"Removed: {result.get('removed', 0)} containers\n"
f"{result.get('message', 'All resources cleaned up')}",
title="Cleanup Complete",
border_style="green"
))
else:
console.print(Panel(
f"[yellow]⚠️ Partial cleanup[/yellow]\n\n"
f"{result.get('message', 'Some resources may still exist')}",
title="Cleanup Status",
border_style="yellow"
))
@cli.command("restart")
@click.option(
"--replicas", "-r",
type=int,
help="New replica count (optional)"
)
def restart_cmd(replicas: int):
"""Restart server (stop then start with same config).
Preserves existing configuration unless overridden with options.
Useful for applying image updates or recovering from errors.
Examples:
# Restart with same configuration
cnode restart
# Restart and change replica count
cnode restart --replicas 5
"""
manager = ServerManager()
# Get current state
async def _get_status():
return await manager.status()
current = anyio.run(_get_status)
if not current["running"]:
console.print("[yellow]No server is running. Use 'cnode start' instead.[/yellow]")
return
# Extract current config
current_replicas = current.get("replicas", 1)
current_port = current.get("port", 11235)
current_image = current.get("image", "unclecode/crawl4ai:latest")
current_mode = current.get("mode", "auto")
# Override with CLI args
new_replicas = replicas if replicas is not None else current_replicas
console.print(Panel(
f"[cyan]Restarting Crawl4AI Server[/cyan]\n\n"
f"Replicas: [yellow]{current_replicas}[/yellow] → [green]{new_replicas}[/green]\n"
f"Port: [yellow]{current_port}[/yellow]\n"
f"Mode: [yellow]{current_mode}[/yellow]",
title="Server Restart",
border_style="cyan"
))
# Stop current
with console.status("[cyan]Stopping current server..."):
async def _stop_server():
return await manager.stop(remove_volumes=False)
stop_result = anyio.run(_stop_server)
if not stop_result["success"]:
console.print(f"[red]Failed to stop server: {stop_result.get('error')}[/red]")
return
# Start new
with console.status("[cyan]Starting server..."):
async def _start_server():
return await manager.start(
replicas=new_replicas,
mode="auto",
port=current_port,
image=current_image
)
start_result = anyio.run(_start_server)
if start_result["success"]:
console.print(Panel(
f"[green]✓ Server restarted successfully![/green]\n\n"
f"URL: [bold]http://localhost:{current_port}[/bold]",
title="Restart Complete",
border_style="green"
))
else:
console.print(Panel(
f"[red]✗ Failed to restart server[/red]\n\n"
f"{start_result.get('error', 'Unknown error')}",
title="Error",
border_style="red"
))
def main():
"""Entry point for cnode CLI"""
cli()
if __name__ == "__main__":
main()
# Test comment

1154
deploy/docker/server_manager.py Normal file
View File

File diff suppressed because it is too large Load Diff

345
deploy/installer/README.md Normal file
View File

@@ -0,0 +1,345 @@
# Crawl4AI Node Manager (cnode) - Installation & Distribution
This directory contains the standalone `cnode` package and installation scripts for managing Crawl4AI Docker server instances.
## Overview
`cnode` is a fast, lightweight CLI tool for managing Crawl4AI Docker servers. It provides:
- One-command deployment with automatic scaling
- Single container for development (N=1)
- Docker Swarm for production with built-in load balancing (N>1)
- Docker Compose + Nginx as fallback (N>1)
## Directory Structure
```
deploy/installer/
├── README.md # This file
├── cnode_pkg/ # Standalone Python package
│ ├── __init__.py # Package marker
│ ├── cli.py # CLI interface (commands)
│ ├── server_manager.py # Docker orchestration logic
│ └── requirements.txt # Python dependencies
├── install-cnode.sh # Local installation script
├── deploy.sh # Remote installation script (for users)
└── releases/ # Release artifacts for distribution
```
## Installation
### For Users (Remote Installation)
Users can install `cnode` directly from the web:
```bash
# Install from GitHub/website
curl -sSL https://crawl4ai.com/install-cnode.sh | bash
# Or with wget
wget -qO- https://crawl4ai.com/install-cnode.sh | bash
```
### For Local Testing
Test the installation locally:
```bash
cd deploy/installer
./install-cnode.sh
```
## Package Contents
### `cnode_pkg/` - Python Package
This is a self-contained Python package with:
- **`cli.py`**: Click-based CLI with all commands (start, stop, status, scale, logs, cleanup, restart)
- **`server_manager.py`**: Core Docker orchestration logic
- **`requirements.txt`**: Dependencies (click, rich, anyio, pyyaml)
- **`__init__.py`**: Package initialization
### Installation Script
**`install-cnode.sh`** does the following:
1. Checks for Python 3.8+ and pip
2. Checks for Docker (warns if not found)
3. Installs Python dependencies
4. Copies `cnode_pkg/` to `/usr/local/lib/cnode/`
5. Creates wrapper script at `/usr/local/bin/cnode`
6. Verifies installation
### Wrapper Script
Created at `/usr/local/bin/cnode`:
```bash
#!/usr/bin/env bash
set -e
# Find Python
if command -v python3 &> /dev/null; then
PYTHON_CMD="python3"
elif command -v python &> /dev/null; then
PYTHON_CMD="python"
else
echo "Error: Python 3.8+ required" >&2
exit 1
fi
# Run cnode
export PYTHONPATH="/usr/local/lib/cnode:$PYTHONPATH"
exec $PYTHON_CMD -m cnode_pkg.cli "$@"
```
## Performance
**Blazing Fast Startup:**
- **~0.1 seconds** to launch
- 49x faster than compiled binary alternatives
- Minimal overhead, maximum responsiveness
## Requirements
### User Requirements
- Python 3.8 or higher
- pip (Python package manager)
- Docker (for running servers)
### Dependencies (Auto-installed)
- click >= 8.0.0 (CLI framework)
- rich >= 13.0.0 (Terminal formatting)
- anyio >= 3.0.0 (Async I/O)
- pyyaml >= 6.0.0 (YAML parsing)
## Usage
After installation:
```bash
# Quick start
cnode start # Single container on port 11235
cnode start --replicas 5 # 5-replica cluster
cnode status # Check server status
cnode logs -f # Follow logs
cnode scale 10 # Scale to 10 replicas
cnode stop # Stop server
# Get help
cnode --help
cnode start --help
```
## Development Workflow
### Making Changes
1. **Edit source code** in `deploy/docker/`:
```bash
vim deploy/docker/cnode_cli.py
vim deploy/docker/server_manager.py
```
2. **Update package** by copying to installer:
```bash
# Copy CLI
cp deploy/docker/cnode_cli.py deploy/installer/cnode_pkg/cli.py
# Fix imports (deploy.docker → cnode_pkg)
sed -i 's/from deploy\.docker\./from cnode_pkg./g' deploy/installer/cnode_pkg/cli.py
# Copy server manager
cp deploy/docker/server_manager.py deploy/installer/cnode_pkg/server_manager.py
```
3. **Test locally**:
```bash
cd deploy/installer
./install-cnode.sh
cnode --help
```
4. **Commit both**:
```bash
git add deploy/docker/cnode_cli.py
git add deploy/installer/cnode_pkg/cli.py
git commit -m "Update cnode: [description]"
```
### Creating a Release
1. **Tag the release**:
```bash
git tag -a v1.0.0 -m "Release v1.0.0"
git push origin v1.0.0
```
2. **Package for distribution**:
```bash
cd deploy/installer
tar -czf releases/cnode-v1.0.0.tar.gz cnode_pkg/ install-cnode.sh
```
3. **Create GitHub release**:
```bash
gh release create v1.0.0 \
releases/cnode-v1.0.0.tar.gz \
--title "cnode v1.0.0" \
--notes "Release notes here"
```
4. **Update deployment script** (if needed):
- Update `deploy.sh` with new version/URL
- Upload to hosting (e.g., `https://crawl4ai.com/install-cnode.sh`)
## Deployment
### Remote Installation Script
The `deploy.sh` script is meant to be hosted at a public URL for user installation:
```bash
# Upload to your server
scp deploy.sh user@crawl4ai.com:/var/www/html/install-cnode.sh
# Or use GitHub raw URL
https://raw.githubusercontent.com/unclecode/crawl4ai/main/deploy/installer/deploy.sh
```
Users can then install with:
```bash
curl -sSL https://crawl4ai.com/install-cnode.sh | bash
```
## Backward Compatibility
The main Crawl4AI CLI (`crwl`) includes a redirect for backward compatibility:
```bash
# These work identically:
crwl server start --replicas 3
cnode start --replicas 3
# All subcommands redirect:
crwl server status → cnode status
crwl server stop → cnode stop
crwl server scale 5 → cnode scale 5
crwl server logs -f → cnode logs -f
```
This ensures existing scripts continue working while users migrate to `cnode`.
## Uninstallation
To remove cnode:
```bash
# Remove command
sudo rm /usr/local/bin/cnode
# Remove package
sudo rm -rf /usr/local/lib/cnode
# (Optional) Uninstall dependencies
pip uninstall click rich anyio pyyaml
```
## Troubleshooting
### Python Not Found
```bash
# Install Python 3.8+
# macOS: brew install python3
# Ubuntu: sudo apt install python3 python3-pip
# RHEL/CentOS: sudo yum install python3 python3-pip
```
### Permission Denied
```bash
# Run installer with sudo
sudo ./install-cnode.sh
# Or change install location
INSTALL_DIR=$HOME/.local/bin ./install-cnode.sh
```
### Command Not Found After Install
```bash
# Add to PATH in ~/.bashrc or ~/.zshrc
export PATH="/usr/local/bin:$PATH"
# Reload shell
source ~/.bashrc # or source ~/.zshrc
```
### Dependencies Install Failed
```bash
# Install manually
pip install --user click rich anyio pyyaml
# Or with break-system-packages (if needed)
pip install --user --break-system-packages click rich anyio pyyaml
```
### Docker Not Running
```bash
# macOS: Start Docker Desktop
# Linux: sudo systemctl start docker
# Check Docker
docker --version
docker ps
```
## Architecture
### Component Flow
```
User runs: cnode start
↓
/usr/local/bin/cnode (wrapper script)
↓
Finds python3 executable
↓
Sets PYTHONPATH=/usr/local/lib/cnode
↓
python3 -m cnode_pkg.cli start
↓
cli.py → start_cmd()
↓
server_manager.py → ServerManager.start()
↓
Docker orchestration (single/swarm/compose)
↓
Server running!
```
### Why Python Wrapper vs Binary?
We chose a Python wrapper over compiled binaries (PyInstaller) because:
| Metric | Python Wrapper | PyInstaller Binary |
|--------|---------------|-------------------|
| Startup time | **0.1s** | 4.7s |
| Size | ~50KB wrapper | 8.8MB |
| Updates | Easy (just copy files) | Rebuild required |
| Dependencies | Python 3.8+ | None |
| Platform | Any with Python | OS-specific builds |
Since users running Crawl4AI already have Python, the wrapper is the clear winner.
## Support
For issues or questions:
- GitHub Issues: https://github.com/unclecode/crawl4ai/issues
- Documentation: https://docs.crawl4ai.com
- Discord: https://discord.gg/crawl4ai
## Version History
- **v1.0.0**: Initial release with Python wrapper approach
- Fast startup (~0.1s)
- Supports single container, Docker Swarm, and Compose modes
- Auto-scaling and load balancing
- Real-time monitoring and logs

5
deploy/installer/cnode_pkg/__init__.py Normal file
View File

@@ -0,0 +1,5 @@
"""
Crawl4AI Node Manager (cnode) - Docker server orchestration CLI
"""
__version__ = "1.0.0"

492
deploy/installer/cnode_pkg/cli.py Normal file
View File

@@ -0,0 +1,492 @@
"""
Crawl4AI Server CLI Commands
Provides `cnode` command group for Docker orchestration.
"""
import click
import anyio
from rich.console import Console
from rich.table import Table
from rich.panel import Panel
from rich.prompt import Confirm
from cnode_pkg.server_manager import ServerManager
console = Console()
@click.group()
def cli():
"""Manage Crawl4AI Docker server instances
\b
One-command deployment with automatic scaling:
• Single container for development (N=1)
• Docker Swarm for production with built-in load balancing (N>1)
• Docker Compose + Nginx as fallback (N>1)
\b
Examples:
cnode start # Single container on port 11235
cnode start --replicas 3 # Auto-detect Swarm or Compose
cnode start -r 5 --port 8080 # 5 replicas on custom port
cnode status # Check current deployment
cnode scale 10 # Scale to 10 replicas
cnode stop # Stop and cleanup
"""
pass
@cli.command("start")
@click.option(
"--replicas", "-r",
type=int,
default=1,
help="Number of container replicas (default: 1)"
)
@click.option(
"--mode",
type=click.Choice(["auto", "single", "swarm", "compose"]),
default="auto",
help="Deployment mode (default: auto-detect)"
)
@click.option(
"--port", "-p",
type=int,
default=11235,
help="External port to expose (default: 11235)"
)
@click.option(
"--env-file",
type=click.Path(exists=True),
help="Path to environment file"
)
@click.option(
"--image",
default="unclecode/crawl4ai:latest",
help="Docker image to use (default: unclecode/crawl4ai:latest)"
)
def start_cmd(replicas: int, mode: str, port: int, env_file: str, image: str):
"""Start Crawl4AI server with automatic orchestration.
Deployment modes:
- auto: Automatically choose best mode (default)
- single: Single container (N=1 only)
- swarm: Docker Swarm with built-in load balancing
- compose: Docker Compose + Nginx reverse proxy
The server will:
1. Check if Docker is running
2. Validate port availability
3. Pull image if needed
4. Start container(s) with health checks
5. Save state for management
Examples:
# Development: single container
cnode start
# Production: 5 replicas with Swarm
cnode start --replicas 5
# Custom configuration
cnode start -r 3 --port 8080 --env-file .env.prod
"""
manager = ServerManager()
console.print(Panel(
f"[cyan]Starting Crawl4AI Server[/cyan]\n\n"
f"Replicas: [yellow]{replicas}[/yellow]\n"
f"Mode: [yellow]{mode}[/yellow]\n"
f"Port: [yellow]{port}[/yellow]\n"
f"Image: [yellow]{image}[/yellow]",
title="Server Start",
border_style="cyan"
))
with console.status("[cyan]Starting server..."):
async def _start():
return await manager.start(
replicas=replicas,
mode=mode,
port=port,
env_file=env_file,
image=image
)
result = anyio.run(_start)
if result["success"]:
console.print(Panel(
f"[green]✓ Server started successfully![/green]\n\n"
f"Mode: [cyan]{result.get('state_data', {}).get('mode', mode)}[/cyan]\n"
f"URL: [bold]http://localhost:{port}[/bold]\n"
f"Health: [bold]http://localhost:{port}/health[/bold]\n"
f"Monitor: [bold]http://localhost:{port}/monitor[/bold]",
title="Server Running",
border_style="green"
))
else:
error_msg = result.get("error", result.get("message", "Unknown error"))
console.print(Panel(
f"[red]✗ Failed to start server[/red]\n\n"
f"{error_msg}",
title="Error",
border_style="red"
))
if "already running" in error_msg.lower():
console.print("\n[yellow]Hint: Use 'cnode status' to check current deployment[/yellow]")
console.print("[yellow] Use 'cnode stop' to stop existing server[/yellow]")
@cli.command("status")
def status_cmd():
"""Show current server status and deployment info.
Displays:
- Running state (up/down)
- Deployment mode (single/swarm/compose)
- Number of replicas
- Port mapping
- Uptime
- Image version
Example:
cnode status
"""
manager = ServerManager()
async def _status():
return await manager.status()
result = anyio.run(_status)
if result["running"]:
table = Table(title="Crawl4AI Server Status", border_style="green")
table.add_column("Property", style="cyan")
table.add_column("Value", style="green")
table.add_row("Status", "🟢 Running")
table.add_row("Mode", result["mode"])
table.add_row("Replicas", str(result.get("replicas", 1)))
table.add_row("Port", str(result.get("port", 11235)))
table.add_row("Image", result.get("image", "unknown"))
table.add_row("Uptime", result.get("uptime", "unknown"))
table.add_row("Started", result.get("started_at", "unknown"))
console.print(table)
console.print(f"\n[green]✓ Server is healthy[/green]")
console.print(f"[dim]Access: http://localhost:{result.get('port', 11235)}[/dim]")
else:
console.print(Panel(
f"[yellow]No server is currently running[/yellow]\n\n"
f"Use 'cnode start' to launch a server",
title="Server Status",
border_style="yellow"
))
@cli.command("stop")
@click.option(
"--remove-volumes",
is_flag=True,
help="Remove associated volumes (WARNING: deletes data)"
)
def stop_cmd(remove_volumes: bool):
"""Stop running Crawl4AI server and cleanup resources.
This will:
1. Stop all running containers/services
2. Remove containers
3. Optionally remove volumes (--remove-volumes)
4. Clean up state files
WARNING: Use --remove-volumes with caution as it will delete
persistent data including Redis databases and logs.
Examples:
# Stop server, keep volumes
cnode stop
# Stop and remove all data
cnode stop --remove-volumes
"""
manager = ServerManager()
# Confirm if removing volumes
if remove_volumes:
if not Confirm.ask(
"[red]⚠️ This will delete all server data including Redis databases. Continue?[/red]"
):
console.print("[yellow]Cancelled[/yellow]")
return
with console.status("[cyan]Stopping server..."):
async def _stop():
return await manager.stop(remove_volumes=remove_volumes)
result = anyio.run(_stop)
if result["success"]:
console.print(Panel(
f"[green]✓ Server stopped successfully[/green]\n\n"
f"{result.get('message', 'All resources cleaned up')}",
title="Server Stopped",
border_style="green"
))
else:
console.print(Panel(
f"[red]✗ Error stopping server[/red]\n\n"
f"{result.get('error', result.get('message', 'Unknown error'))}",
title="Error",
border_style="red"
))
@cli.command("scale")
@click.argument("replicas", type=int)
def scale_cmd(replicas: int):
"""Scale server to specified number of replicas.
Only works with Swarm or Compose modes. Single container
mode cannot be scaled (must stop and restart with --replicas).
Scaling is live and does not require downtime. The load
balancer will automatically distribute traffic to new replicas.
Examples:
# Scale up to 10 replicas
cnode scale 10
# Scale down to 2 replicas
cnode scale 2
# Scale to 1 (minimum)
cnode scale 1
"""
if replicas < 1:
console.print("[red]Error: Replicas must be at least 1[/red]")
return
manager = ServerManager()
with console.status(f"[cyan]Scaling to {replicas} replicas..."):
async def _scale():
return await manager.scale(replicas=replicas)
result = anyio.run(_scale)
if result["success"]:
console.print(Panel(
f"[green]✓ Scaled successfully[/green]\n\n"
f"New replica count: [bold]{replicas}[/bold]\n"
f"Mode: [cyan]{result.get('mode')}[/cyan]",
title="Scaling Complete",
border_style="green"
))
else:
error_msg = result.get("error", result.get("message", "Unknown error"))
console.print(Panel(
f"[red]✗ Scaling failed[/red]\n\n"
f"{error_msg}",
title="Error",
border_style="red"
))
if "single container" in error_msg.lower():
console.print("\n[yellow]Hint: For single container mode:[/yellow]")
console.print("[yellow] 1. cnode stop[/yellow]")
console.print(f"[yellow] 2. cnode start --replicas {replicas}[/yellow]")
@cli.command("logs")
@click.option(
"--follow", "-f",
is_flag=True,
help="Follow log output (like tail -f)"
)
@click.option(
"--tail",
type=int,
default=100,
help="Number of lines to show (default: 100)"
)
def logs_cmd(follow: bool, tail: int):
"""View server logs.
Shows logs from running containers/services. Use --follow
to stream logs in real-time.
Examples:
# Show last 100 lines
cnode logs
# Show last 500 lines
cnode logs --tail 500
# Follow logs in real-time
cnode logs --follow
# Combine options
cnode logs -f --tail 50
"""
manager = ServerManager()
async def _logs():
return await manager.logs(follow=follow, tail=tail)
output = anyio.run(_logs)
console.print(output)
@cli.command("cleanup")
@click.option(
"--force",
is_flag=True,
help="Force cleanup even if state file doesn't exist"
)
def cleanup_cmd(force: bool):
"""Force cleanup of all Crawl4AI Docker resources.
Stops and removes all containers, networks, and optionally volumes.
Useful when server is stuck or state is corrupted.
Examples:
# Clean up everything
cnode cleanup
# Force cleanup (ignore state file)
cnode cleanup --force
"""
manager = ServerManager()
console.print(Panel(
f"[yellow]⚠️ Cleaning up Crawl4AI Docker resources[/yellow]\n\n"
f"This will stop and remove:\n"
f"- All Crawl4AI containers\n"
f"- Nginx load balancer\n"
f"- Redis instance\n"
f"- Docker networks\n"
f"- State files",
title="Cleanup",
border_style="yellow"
))
if not force and not Confirm.ask("[yellow]Continue with cleanup?[/yellow]"):
console.print("[yellow]Cancelled[/yellow]")
return
with console.status("[cyan]Cleaning up resources..."):
async def _cleanup():
return await manager.cleanup(force=force)
result = anyio.run(_cleanup)
if result["success"]:
console.print(Panel(
f"[green]✓ Cleanup completed successfully[/green]\n\n"
f"Removed: {result.get('removed', 0)} containers\n"
f"{result.get('message', 'All resources cleaned up')}",
title="Cleanup Complete",
border_style="green"
))
else:
console.print(Panel(
f"[yellow]⚠️ Partial cleanup[/yellow]\n\n"
f"{result.get('message', 'Some resources may still exist')}",
title="Cleanup Status",
border_style="yellow"
))
@cli.command("restart")
@click.option(
"--replicas", "-r",
type=int,
help="New replica count (optional)"
)
def restart_cmd(replicas: int):
"""Restart server (stop then start with same config).
Preserves existing configuration unless overridden with options.
Useful for applying image updates or recovering from errors.
Examples:
# Restart with same configuration
cnode restart
# Restart and change replica count
cnode restart --replicas 5
"""
manager = ServerManager()
# Get current state
async def _get_status():
return await manager.status()
current = anyio.run(_get_status)
if not current["running"]:
console.print("[yellow]No server is running. Use 'cnode start' instead.[/yellow]")
return
# Extract current config
current_replicas = current.get("replicas", 1)
current_port = current.get("port", 11235)
current_image = current.get("image", "unclecode/crawl4ai:latest")
current_mode = current.get("mode", "auto")
# Override with CLI args
new_replicas = replicas if replicas is not None else current_replicas
console.print(Panel(
f"[cyan]Restarting Crawl4AI Server[/cyan]\n\n"
f"Replicas: [yellow]{current_replicas}[/yellow] → [green]{new_replicas}[/green]\n"
f"Port: [yellow]{current_port}[/yellow]\n"
f"Mode: [yellow]{current_mode}[/yellow]",
title="Server Restart",
border_style="cyan"
))
# Stop current
with console.status("[cyan]Stopping current server..."):
async def _stop_server():
return await manager.stop(remove_volumes=False)
stop_result = anyio.run(_stop_server)
if not stop_result["success"]:
console.print(f"[red]Failed to stop server: {stop_result.get('error')}[/red]")
return
# Start new
with console.status("[cyan]Starting server..."):
async def _start_server():
return await manager.start(
replicas=new_replicas,
mode="auto",
port=current_port,
image=current_image
)
start_result = anyio.run(_start_server)
if start_result["success"]:
console.print(Panel(
f"[green]✓ Server restarted successfully![/green]\n\n"
f"URL: [bold]http://localhost:{current_port}[/bold]",
title="Restart Complete",
border_style="green"
))
else:
console.print(Panel(
f"[red]✗ Failed to restart server[/red]\n\n"
f"{start_result.get('error', 'Unknown error')}",
title="Error",
border_style="red"
))
def main():
"""Entry point for cnode CLI"""
cli()
if __name__ == "__main__":
main()
# Test comment

4
deploy/installer/cnode_pkg/requirements.txt Normal file
View File

@@ -0,0 +1,4 @@
click>=8.0.0
rich>=13.0.0
anyio>=3.0.0
pyyaml>=6.0.0

1154
deploy/installer/cnode_pkg/server_manager.py Normal file
View File

File diff suppressed because it is too large Load Diff

217
deploy/installer/deploy.sh Executable file
View File

@@ -0,0 +1,217 @@
#!/bin/bash
# Crawl4AI Node Manager (cnode) Installation Script
# Usage: curl -sSL https://crawl4ai.com/deploy.sh | bash
# Or: wget -qO- https://crawl4ai.com/deploy.sh | bash
set -e
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Configuration
INSTALL_DIR="${INSTALL_DIR:-/usr/local/bin}"
BINARY_NAME="cnode"
GITHUB_REPO="unclecode/crawl4ai"
RELEASE_TAG="${CNODE_VERSION:-latest}"
echo -e "${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Crawl4AI Node Manager (cnode) Installation Script ║${NC}"
echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}\n"
# Detect OS and architecture
detect_platform() {
OS="$(uname -s)"
ARCH="$(uname -m)"
case "$OS" in
Linux*)
OS_TYPE="linux"
;;
Darwin*)
OS_TYPE="macos"
;;
*)
echo -e "${RED}Error: Unsupported operating system: $OS${NC}"
exit 1
;;
esac
case "$ARCH" in
x86_64|amd64)
ARCH_TYPE="amd64"
;;
aarch64|arm64)
ARCH_TYPE="arm64"
;;
*)
echo -e "${RED}Error: Unsupported architecture: $ARCH${NC}"
exit 1
;;
esac
echo -e "${BLUE}Detected platform: ${YELLOW}$OS_TYPE-$ARCH_TYPE${NC}"
}
# Check if Docker is installed
check_docker() {
if ! command -v docker &> /dev/null; then
echo -e "${YELLOW}⚠️ Docker not found. cnode requires Docker to manage server instances.${NC}"
echo -e "${YELLOW}Install Docker from: https://docs.docker.com/get-docker/${NC}\n"
read -p "Continue installation anyway? (y/N) " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
exit 1
fi
else
echo -e "${GREEN}✓ Docker is installed${NC}"
fi
}
# Check write permissions
check_permissions() {
if [ ! -w "$INSTALL_DIR" ]; then
echo -e "${YELLOW}⚠️ No write permission for $INSTALL_DIR${NC}"
echo -e "${YELLOW}The script will attempt to use sudo for installation.${NC}\n"
USE_SUDO="sudo"
else
USE_SUDO=""
fi
}
# Download binary
download_binary() {
BINARY_URL="https://github.com/$GITHUB_REPO/releases/download/$RELEASE_TAG/cnode-$OS_TYPE-$ARCH_TYPE"
echo -e "${BLUE}Downloading cnode from GitHub...${NC}"
echo -e "${YELLOW}URL: $BINARY_URL${NC}\n"
# Create temp directory
TMP_DIR="$(mktemp -d)"
TMP_FILE="$TMP_DIR/$BINARY_NAME"
# Download with curl or wget
if command -v curl &> /dev/null; then
if ! curl -fSL "$BINARY_URL" -o "$TMP_FILE"; then
echo -e "${RED}Error: Failed to download binary${NC}"
echo -e "${YELLOW}URL: $BINARY_URL${NC}"
rm -rf "$TMP_DIR"
exit 1
fi
elif command -v wget &> /dev/null; then
if ! wget -q "$BINARY_URL" -O "$TMP_FILE"; then
echo -e "${RED}Error: Failed to download binary${NC}"
echo -e "${YELLOW}URL: $BINARY_URL${NC}"
rm -rf "$TMP_DIR"
exit 1
fi
else
echo -e "${RED}Error: Neither curl nor wget found${NC}"
echo -e "${YELLOW}Please install curl or wget and try again${NC}"
rm -rf "$TMP_DIR"
exit 1
fi
# Make executable
chmod +x "$TMP_FILE"
echo "$TMP_FILE"
}
# Install binary
install_binary() {
local tmp_file="$1"
local install_path="$INSTALL_DIR/$BINARY_NAME"
echo -e "\n${BLUE}Installing cnode to $install_path...${NC}"
# Check if already installed
if [ -f "$install_path" ]; then
echo -e "${YELLOW}⚠️ cnode is already installed${NC}"
read -p "Overwrite existing installation? (y/N) " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
echo -e "${YELLOW}Installation cancelled${NC}"
rm -rf "$(dirname "$tmp_file")"
exit 0
fi
fi
# Install
if ! $USE_SUDO mv "$tmp_file" "$install_path"; then
echo -e "${RED}Error: Failed to install binary${NC}"
rm -rf "$(dirname "$tmp_file")"
exit 1
fi
# Cleanup temp directory
rm -rf "$(dirname "$tmp_file")"
echo -e "${GREEN}✓ Installation successful${NC}"
}
# Verify installation
verify_installation() {
echo -e "\n${BLUE}Verifying installation...${NC}"
if ! command -v $BINARY_NAME &> /dev/null; then
echo -e "${RED}Error: $BINARY_NAME not found in PATH${NC}"
echo -e "${YELLOW}You may need to add $INSTALL_DIR to your PATH${NC}"
echo -e "${YELLOW}Add this to your ~/.bashrc or ~/.zshrc:${NC}"
echo -e "${YELLOW}export PATH=\"$INSTALL_DIR:\$PATH\"${NC}\n"
exit 1
fi
# Test version
if $BINARY_NAME --help &> /dev/null; then
echo -e "${GREEN}✓ $BINARY_NAME is working correctly${NC}"
else
echo -e "${RED}Error: $BINARY_NAME failed to execute${NC}"
exit 1
fi
}
# Show completion message
show_completion() {
local version
version=$($BINARY_NAME --help | head -1 || echo "unknown")
echo -e "\n${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Installation Complete! ║${NC}"
echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}\n"
echo -e "${BLUE}cnode is now installed and ready to use!${NC}\n"
echo -e "${YELLOW}Quick Start:${NC}"
echo -e " ${GREEN}cnode start${NC} # Start single server instance"
echo -e " ${GREEN}cnode start --replicas 5${NC} # Start 5-replica cluster"
echo -e " ${GREEN}cnode status${NC} # Check server status"
echo -e " ${GREEN}cnode logs -f${NC} # Follow server logs"
echo -e " ${GREEN}cnode scale 10${NC} # Scale to 10 replicas"
echo -e " ${GREEN}cnode stop${NC} # Stop server"
echo -e "\n${YELLOW}For more information:${NC}"
echo -e " ${BLUE}cnode --help${NC}"
echo -e " ${BLUE}https://github.com/$GITHUB_REPO${NC}\n"
}
# Main installation flow
main() {
detect_platform
check_docker
check_permissions
# Download and install
TMP_FILE=$(download_binary)
install_binary "$TMP_FILE"
# Verify
verify_installation
show_completion
}
# Run installation
main

142
deploy/installer/install-cnode.sh Executable file
View File

@@ -0,0 +1,142 @@
#!/bin/bash
# Crawl4AI Node Manager (cnode) - Fast Installation Script
# This installs cnode as a Python package with a wrapper script
set -e
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'
echo -e "${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Crawl4AI Node Manager (cnode) - Fast Installer ║${NC}"
echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}\n"
# Configuration
INSTALL_DIR="${INSTALL_DIR:-/usr/local/bin}"
LIB_DIR="${LIB_DIR:-/usr/local/lib/cnode}"
# Check Python
echo -e "${BLUE}Checking Python installation...${NC}"
if command -v python3 &> /dev/null; then
PYTHON_CMD="python3"
elif command -v python &> /dev/null; then
PYTHON_CMD="python"
else
echo -e "${RED}Error: Python 3.8+ is required but not found${NC}"
echo -e "${YELLOW}Install from: https://www.python.org/downloads/${NC}"
exit 1
fi
# Check Python version
PYTHON_VERSION=$($PYTHON_CMD -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
echo -e "${GREEN}✓ Found Python $PYTHON_VERSION${NC}"
if [ "$(printf '%s\n' "3.8" "$PYTHON_VERSION" | sort -V | head -n1)" != "3.8" ]; then
echo -e "${RED}Error: Python 3.8+ required, found $PYTHON_VERSION${NC}"
exit 1
fi
# Check pip
if ! $PYTHON_CMD -m pip --version &> /dev/null; then
echo -e "${RED}Error: pip is required${NC}"
echo -e "${YELLOW}Install pip: $PYTHON_CMD -m ensurepip${NC}"
exit 1
fi
echo -e "${GREEN}✓ pip is available${NC}"
# Check Docker
echo -e "\n${BLUE}Checking Docker...${NC}"
if ! command -v docker &> /dev/null; then
echo -e "${YELLOW}⚠️ Docker not found (required for running servers)${NC}"
echo -e "${YELLOW}Install from: https://docs.docker.com/get-docker/${NC}\n"
else
echo -e "${GREEN}✓ Docker is installed${NC}"
fi
# Check permissions
USE_SUDO=""
if [ ! -w "$INSTALL_DIR" ] || [ ! -w "/usr/local" ]; then
echo -e "\n${YELLOW}⚠️ Root permission required for installation${NC}"
USE_SUDO="sudo"
fi
# Get script directory
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Install Python dependencies
echo -e "\n${BLUE}Installing Python dependencies...${NC}"
$PYTHON_CMD -m pip install --quiet --user -r "$SCRIPT_DIR/cnode_pkg/requirements.txt" 2>/dev/null || \
$PYTHON_CMD -m pip install --quiet --user --break-system-packages -r "$SCRIPT_DIR/cnode_pkg/requirements.txt" 2>/dev/null || {
echo -e "${YELLOW}⚠️ Could not install dependencies with pip${NC}"
echo -e "${YELLOW}Trying to continue anyway (dependencies may already be installed)${NC}"
}
echo -e "${GREEN}✓ Dependencies check complete${NC}"
# Create lib directory
echo -e "\n${BLUE}Installing cnode package...${NC}"
$USE_SUDO mkdir -p "$LIB_DIR"
$USE_SUDO cp -r "$SCRIPT_DIR/cnode_pkg" "$LIB_DIR/"
echo -e "${GREEN}✓ Package installed to $LIB_DIR${NC}"
# Create wrapper script
echo -e "\n${BLUE}Creating cnode command...${NC}"
$USE_SUDO tee "$INSTALL_DIR/cnode" > /dev/null << 'EOF'
#!/usr/bin/env bash
# Crawl4AI Node Manager (cnode) wrapper
set -e
# Find Python
if command -v python3 &> /dev/null; then
PYTHON_CMD="python3"
elif command -v python &> /dev/null; then
PYTHON_CMD="python"
else
echo "Error: Python 3.8+ required" >&2
exit 1
fi
# Add cnode to Python path and run
export PYTHONPATH="/usr/local/lib/cnode:$PYTHONPATH"
exec $PYTHON_CMD -m cnode_pkg.cli "$@"
EOF
$USE_SUDO chmod +x "$INSTALL_DIR/cnode"
echo -e "${GREEN}✓ cnode command created${NC}"
# Verify installation
echo -e "\n${BLUE}Verifying installation...${NC}"
if ! command -v cnode &> /dev/null; then
echo -e "${RED}Error: cnode not found in PATH${NC}"
echo -e "${YELLOW}Add $INSTALL_DIR to your PATH${NC}"
exit 1
fi
if ! cnode --help &> /dev/null; then
echo -e "${RED}Error: cnode command failed${NC}"
exit 1
fi
echo -e "${GREEN}✓ Installation verified${NC}"
# Success message
echo -e "\n${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Installation Complete! ║${NC}"
echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}\n"
echo -e "${BLUE}cnode is now installed and ready!${NC}\n"
echo -e "${YELLOW}Quick Start:${NC}"
echo -e " ${GREEN}cnode start${NC} # Start single server"
echo -e " ${GREEN}cnode start --replicas 5${NC} # Start 5-replica cluster"
echo -e " ${GREEN}cnode status${NC} # Check status"
echo -e " ${GREEN}cnode logs -f${NC} # Follow logs"
echo -e " ${GREEN}cnode stop${NC} # Stop server"
echo -e "\n${YELLOW}More help:${NC}"
echo -e " ${BLUE}cnode --help${NC}\n"

33
deploy/installer/sync-cnode.sh Executable file
View File

@@ -0,0 +1,33 @@
#!/bin/bash
# Sync cnode source code to installer package
# Run this before committing changes to cnode
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SOURCE_DIR="$SCRIPT_DIR/../docker"
PKG_DIR="$SCRIPT_DIR/cnode_pkg"
echo "🔄 Syncing cnode source to package..."
# Copy CLI
echo " → Copying cnode_cli.py to cli.py"
cp "$SOURCE_DIR/cnode_cli.py" "$PKG_DIR/cli.py"
# Fix imports
echo " → Fixing imports (deploy.docker → cnode_pkg)"
sed -i '' 's/from deploy\.docker\./from cnode_pkg./g' "$PKG_DIR/cli.py"
# Copy server manager
echo " → Copying server_manager.py"
cp "$SOURCE_DIR/server_manager.py" "$PKG_DIR/server_manager.py"
echo "✅ Sync complete!"
echo ""
echo "Files updated:"
echo " • deploy/installer/cnode_pkg/cli.py"
echo " • deploy/installer/cnode_pkg/server_manager.py"
echo ""
echo "Next steps:"
echo " 1. Test: cd deploy/installer && ./install-cnode.sh"
echo " 2. Commit both source and package files"

24
setup-hooks.sh Executable file
View File

@@ -0,0 +1,24 @@
#!/bin/bash
# Setup Git hooks for cnode auto-sync
# Run this once after cloning the repo: ./setup-hooks.sh
set -e
echo "🔧 Setting up Git hooks..."
# Configure Git to use .githooks directory
git config core.hooksPath .githooks
echo "✅ Git hooks configured!"
echo ""
echo "Hooks installed:"
echo " • pre-commit: Auto-syncs cnode source → package when committing"
echo ""
echo "What this means:"
echo " ✅ Edit deploy/docker/cnode_cli.py"
echo " ✅ Run: git add deploy/docker/cnode_cli.py"
echo " ✅ Run: git commit -m \"update cnode\""
echo " ✅ Hook automatically syncs to deploy/installer/cnode_pkg/"
echo " ✅ Synced files are auto-staged in the same commit"
echo ""
echo "You're all set! 🚀"

14
tests/cli/test_cli.py
View File

@@ -65,14 +65,14 @@ class TestCLIBasics:
assert 'Crawl4AI CLI' in result.output
def test_examples(self, runner):
result = runner.invoke(cli, ['--example'])
result = runner.invoke(cli, ['examples'])
assert result.exit_code == 0
assert 'Examples' in result.output
def test_missing_url(self, runner):
result = runner.invoke(cli)
result = runner.invoke(cli, ['crawl'])
assert result.exit_code != 0
assert 'URL argument is required' in result.output
assert ('Missing argument' in result.output or 'required' in result.output.lower())
class TestConfigParsing:
def test_parse_key_values_basic(self):
@@ -101,18 +101,19 @@ class TestConfigLoading:
class TestLLMConfig:
def test_llm_config_creation(self, temp_config_dir, runner):
def input_simulation(inputs):
return runner.invoke(cli, ['https://example.com', '-q', 'test question'],
return runner.invoke(cli, ['crawl', 'https://example.com', '-q', 'test question'],
input='\n'.join(inputs))
class TestCrawlingFeatures:
def test_basic_crawl(self, runner):
result = runner.invoke(cli, ['https://example.com'])
result = runner.invoke(cli, ['crawl', 'https://example.com'])
assert result.exit_code == 0
class TestErrorHandling:
def test_invalid_config_file(self, runner):
result = runner.invoke(cli, [
'crawl',
'https://example.com',
'--browser-config', 'nonexistent.yml'
])
@@ -122,8 +123,9 @@ class TestErrorHandling:
invalid_schema = temp_config_dir / 'invalid_schema.json'
with open(invalid_schema, 'w') as f:
f.write('invalid json')
result = runner.invoke(cli, [
'crawl',
'https://example.com',
'--schema', str(invalid_schema)
])