feat(cnode): add standalone CLI for Docker server management

- Reorganized server management code:
  - Moved server_cli.py -> deploy/docker/cnode_cli.py
  - Moved server_manager.py -> deploy/docker/server_manager.py

- Created fast Python-based installation (0.1s startup):
  - deploy/installer/cnode_pkg/ - Standalone package
  - deploy/installer/install-cnode.sh - Local installer
  - deploy/installer/deploy.sh - Remote installer for users

- Added backward compatibility:
  - crawl4ai/cli.py: 'crwl server' redirects to 'cnode'
  - Updated tests to match new CLI structure (12/12 passing)

- Automated sync workflow:
  - .githooks/pre-commit - Auto-syncs source to package
  - setup-hooks.sh - One-time setup for contributors
  - deploy/installer/sync-cnode.sh - Manual sync script

Performance:
  - Startup time: 0.1s (49x faster than PyInstaller)
  - Size: ~50KB wrapper vs 8.8MB binary

Commands:
  cnode start [--replicas N]  # Start server/cluster
  cnode status                # Check status
  cnode scale N               # Scale replicas
  cnode logs [-f]             # View logs
  cnode stop                  # Stop server
This commit is contained in:
unclecode
2025-10-21 09:31:18 +08:00
parent 342fc52b47
commit cd02616218
16 changed files with 4181 additions and 11 deletions

31
.githooks/pre-commit Executable file
View File

@@ -0,0 +1,31 @@
#!/bin/bash
# Pre-commit hook: Auto-sync cnode files when cnode source is modified
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'
# Check if cnode source files are being committed
CNODE_FILES_CHANGED=$(git diff --cached --name-only | grep -E "deploy/docker/(cnode_cli|server_manager)\.py")
if [ -n "$CNODE_FILES_CHANGED" ]; then
echo -e "${YELLOW}🔄 cnode source files modified, auto-syncing to package...${NC}"
# Run sync script
if [ -f "deploy/installer/sync-cnode.sh" ]; then
bash deploy/installer/sync-cnode.sh
# Stage the synced files
git add deploy/installer/cnode_pkg/cli.py
git add deploy/installer/cnode_pkg/server_manager.py
echo -e "${GREEN}✅ cnode package synced and staged${NC}"
else
echo -e "${RED}❌ Error: sync-cnode.sh not found${NC}"
exit 1
fi
fi
exit 0

3
.gitignore vendored
View File

@@ -185,7 +185,8 @@ Crawl4AI.egg-info/
requirements0.txt requirements0.txt
a.txt a.txt
*.sh # Ignore shell scripts globally, but allow test scripts
# *.sh
.idea .idea
docs/examples/.chainlit/ docs/examples/.chainlit/
docs/examples/.chainlit/* docs/examples/.chainlit/*

View File

@@ -2,6 +2,8 @@ import click
import os import os
import sys import sys
import time import time
import subprocess
import shutil
import humanize import humanize
from typing import Dict, Any, Optional, List from typing import Dict, Any, Optional, List
@@ -626,8 +628,73 @@ def cli():
# Register server command group (Docker orchestration) # Register server command group (Docker orchestration)
from crawl4ai.server_cli import server_cmd # Redirect to standalone 'cnode' CLI
cli.add_command(server_cmd) @cli.command("server", context_settings=dict(
ignore_unknown_options=True,
allow_extra_args=True,
allow_interspersed_args=False
))
@click.pass_context
def server_cmd(ctx):
"""Manage Crawl4AI Docker server instances (deprecated - use 'cnode')
This command has been moved to a standalone CLI called 'cnode'.
For new installations, use:
curl -sSL https://crawl4ai.com/deploy.sh | bash
This redirect allows existing scripts to continue working.
Available commands: start, stop, status, scale, logs
Use 'crwl server <command> --help' for command-specific help.
"""
# Check if cnode is installed
cnode_path = shutil.which("cnode")
# Get all the args (subcommand + options)
args = ctx.args
if not cnode_path:
console.print(Panel(
"[yellow]The 'crwl server' command has been moved to a standalone CLI.[/yellow]\n\n"
"Please install 'cnode' (Crawl4AI Node Manager):\n"
"[cyan]curl -sSL https://crawl4ai.com/deploy.sh | bash[/cyan]\n\n"
"After installation, use:\n"
"[green]cnode <command>[/green] instead of [dim]crwl server <command>[/dim]\n\n"
"For backward compatibility, we're using the local version for now.",
title="Server Command Moved",
border_style="yellow"
))
# Try to use local version
try:
import sys
# Add deploy/docker to path
deploy_path = str(Path(__file__).parent.parent / 'deploy' / 'docker')
if deploy_path not in sys.path:
sys.path.insert(0, deploy_path)
from cnode_cli import cli as cnode_cli
# Forward to cnode with the args
sys.argv = ['cnode'] + args
cnode_cli(standalone_mode=False)
sys.exit(0)
except SystemExit as e:
# Normal exit from click
sys.exit(e.code if hasattr(e, 'code') else 0)
except Exception as e:
console.print(f"[red]Error: Could not find cnode or local server CLI: {e}[/red]")
console.print(f"[dim]Details: {e}[/dim]")
import traceback
console.print(f"[dim]{traceback.format_exc()}[/dim]")
sys.exit(1)
# cnode is installed - forward everything to it
try:
result = subprocess.run([cnode_path] + args, check=False)
sys.exit(result.returncode)
except Exception as e:
console.print(f"[red]Error running cnode: {e}[/red]")
sys.exit(1)
@cli.group("browser") @cli.group("browser")
@@ -1467,9 +1534,15 @@ def default(url: str, example: bool, browser_config: str, crawler_config: str, f
def main(): def main():
import sys import sys
if len(sys.argv) < 2 or sys.argv[1] not in cli.commands: # Don't auto-insert 'crawl' if the command is recognized
if len(sys.argv) >= 2 and sys.argv[1] in cli.commands:
cli()
elif len(sys.argv) < 2:
cli()
else:
# Unknown command - insert 'crawl' for backward compat
sys.argv.insert(1, "crawl") sys.argv.insert(1, "crawl")
cli() cli()
if __name__ == "__main__": if __name__ == "__main__":
main() main()

View File

@@ -0,0 +1 @@
# Deploy docker module

492
deploy/docker/cnode_cli.py Normal file
View File

@@ -0,0 +1,492 @@
"""
Crawl4AI Server CLI Commands
Provides `cnode` command group for Docker orchestration.
"""
import click
import anyio
from rich.console import Console
from rich.table import Table
from rich.panel import Panel
from rich.prompt import Confirm
from deploy.docker.server_manager import ServerManager
console = Console()
@click.group()
def cli():
"""Manage Crawl4AI Docker server instances
\b
One-command deployment with automatic scaling:
• Single container for development (N=1)
• Docker Swarm for production with built-in load balancing (N>1)
• Docker Compose + Nginx as fallback (N>1)
\b
Examples:
cnode start # Single container on port 11235
cnode start --replicas 3 # Auto-detect Swarm or Compose
cnode start -r 5 --port 8080 # 5 replicas on custom port
cnode status # Check current deployment
cnode scale 10 # Scale to 10 replicas
cnode stop # Stop and cleanup
"""
pass
@cli.command("start")
@click.option(
"--replicas", "-r",
type=int,
default=1,
help="Number of container replicas (default: 1)"
)
@click.option(
"--mode",
type=click.Choice(["auto", "single", "swarm", "compose"]),
default="auto",
help="Deployment mode (default: auto-detect)"
)
@click.option(
"--port", "-p",
type=int,
default=11235,
help="External port to expose (default: 11235)"
)
@click.option(
"--env-file",
type=click.Path(exists=True),
help="Path to environment file"
)
@click.option(
"--image",
default="unclecode/crawl4ai:latest",
help="Docker image to use (default: unclecode/crawl4ai:latest)"
)
def start_cmd(replicas: int, mode: str, port: int, env_file: str, image: str):
"""Start Crawl4AI server with automatic orchestration.
Deployment modes:
- auto: Automatically choose best mode (default)
- single: Single container (N=1 only)
- swarm: Docker Swarm with built-in load balancing
- compose: Docker Compose + Nginx reverse proxy
The server will:
1. Check if Docker is running
2. Validate port availability
3. Pull image if needed
4. Start container(s) with health checks
5. Save state for management
Examples:
# Development: single container
cnode start
# Production: 5 replicas with Swarm
cnode start --replicas 5
# Custom configuration
cnode start -r 3 --port 8080 --env-file .env.prod
"""
manager = ServerManager()
console.print(Panel(
f"[cyan]Starting Crawl4AI Server[/cyan]\n\n"
f"Replicas: [yellow]{replicas}[/yellow]\n"
f"Mode: [yellow]{mode}[/yellow]\n"
f"Port: [yellow]{port}[/yellow]\n"
f"Image: [yellow]{image}[/yellow]",
title="Server Start",
border_style="cyan"
))
with console.status("[cyan]Starting server..."):
async def _start():
return await manager.start(
replicas=replicas,
mode=mode,
port=port,
env_file=env_file,
image=image
)
result = anyio.run(_start)
if result["success"]:
console.print(Panel(
f"[green]✓ Server started successfully![/green]\n\n"
f"Mode: [cyan]{result.get('state_data', {}).get('mode', mode)}[/cyan]\n"
f"URL: [bold]http://localhost:{port}[/bold]\n"
f"Health: [bold]http://localhost:{port}/health[/bold]\n"
f"Monitor: [bold]http://localhost:{port}/monitor[/bold]",
title="Server Running",
border_style="green"
))
else:
error_msg = result.get("error", result.get("message", "Unknown error"))
console.print(Panel(
f"[red]✗ Failed to start server[/red]\n\n"
f"{error_msg}",
title="Error",
border_style="red"
))
if "already running" in error_msg.lower():
console.print("\n[yellow]Hint: Use 'cnode status' to check current deployment[/yellow]")
console.print("[yellow] Use 'cnode stop' to stop existing server[/yellow]")
@cli.command("status")
def status_cmd():
"""Show current server status and deployment info.
Displays:
- Running state (up/down)
- Deployment mode (single/swarm/compose)
- Number of replicas
- Port mapping
- Uptime
- Image version
Example:
cnode status
"""
manager = ServerManager()
async def _status():
return await manager.status()
result = anyio.run(_status)
if result["running"]:
table = Table(title="Crawl4AI Server Status", border_style="green")
table.add_column("Property", style="cyan")
table.add_column("Value", style="green")
table.add_row("Status", "🟢 Running")
table.add_row("Mode", result["mode"])
table.add_row("Replicas", str(result.get("replicas", 1)))
table.add_row("Port", str(result.get("port", 11235)))
table.add_row("Image", result.get("image", "unknown"))
table.add_row("Uptime", result.get("uptime", "unknown"))
table.add_row("Started", result.get("started_at", "unknown"))
console.print(table)
console.print(f"\n[green]✓ Server is healthy[/green]")
console.print(f"[dim]Access: http://localhost:{result.get('port', 11235)}[/dim]")
else:
console.print(Panel(
f"[yellow]No server is currently running[/yellow]\n\n"
f"Use 'cnode start' to launch a server",
title="Server Status",
border_style="yellow"
))
@cli.command("stop")
@click.option(
"--remove-volumes",
is_flag=True,
help="Remove associated volumes (WARNING: deletes data)"
)
def stop_cmd(remove_volumes: bool):
"""Stop running Crawl4AI server and cleanup resources.
This will:
1. Stop all running containers/services
2. Remove containers
3. Optionally remove volumes (--remove-volumes)
4. Clean up state files
WARNING: Use --remove-volumes with caution as it will delete
persistent data including Redis databases and logs.
Examples:
# Stop server, keep volumes
cnode stop
# Stop and remove all data
cnode stop --remove-volumes
"""
manager = ServerManager()
# Confirm if removing volumes
if remove_volumes:
if not Confirm.ask(
"[red]⚠️ This will delete all server data including Redis databases. Continue?[/red]"
):
console.print("[yellow]Cancelled[/yellow]")
return
with console.status("[cyan]Stopping server..."):
async def _stop():
return await manager.stop(remove_volumes=remove_volumes)
result = anyio.run(_stop)
if result["success"]:
console.print(Panel(
f"[green]✓ Server stopped successfully[/green]\n\n"
f"{result.get('message', 'All resources cleaned up')}",
title="Server Stopped",
border_style="green"
))
else:
console.print(Panel(
f"[red]✗ Error stopping server[/red]\n\n"
f"{result.get('error', result.get('message', 'Unknown error'))}",
title="Error",
border_style="red"
))
@cli.command("scale")
@click.argument("replicas", type=int)
def scale_cmd(replicas: int):
"""Scale server to specified number of replicas.
Only works with Swarm or Compose modes. Single container
mode cannot be scaled (must stop and restart with --replicas).
Scaling is live and does not require downtime. The load
balancer will automatically distribute traffic to new replicas.
Examples:
# Scale up to 10 replicas
cnode scale 10
# Scale down to 2 replicas
cnode scale 2
# Scale to 1 (minimum)
cnode scale 1
"""
if replicas < 1:
console.print("[red]Error: Replicas must be at least 1[/red]")
return
manager = ServerManager()
with console.status(f"[cyan]Scaling to {replicas} replicas..."):
async def _scale():
return await manager.scale(replicas=replicas)
result = anyio.run(_scale)
if result["success"]:
console.print(Panel(
f"[green]✓ Scaled successfully[/green]\n\n"
f"New replica count: [bold]{replicas}[/bold]\n"
f"Mode: [cyan]{result.get('mode')}[/cyan]",
title="Scaling Complete",
border_style="green"
))
else:
error_msg = result.get("error", result.get("message", "Unknown error"))
console.print(Panel(
f"[red]✗ Scaling failed[/red]\n\n"
f"{error_msg}",
title="Error",
border_style="red"
))
if "single container" in error_msg.lower():
console.print("\n[yellow]Hint: For single container mode:[/yellow]")
console.print("[yellow] 1. cnode stop[/yellow]")
console.print(f"[yellow] 2. cnode start --replicas {replicas}[/yellow]")
@cli.command("logs")
@click.option(
"--follow", "-f",
is_flag=True,
help="Follow log output (like tail -f)"
)
@click.option(
"--tail",
type=int,
default=100,
help="Number of lines to show (default: 100)"
)
def logs_cmd(follow: bool, tail: int):
"""View server logs.
Shows logs from running containers/services. Use --follow
to stream logs in real-time.
Examples:
# Show last 100 lines
cnode logs
# Show last 500 lines
cnode logs --tail 500
# Follow logs in real-time
cnode logs --follow
# Combine options
cnode logs -f --tail 50
"""
manager = ServerManager()
async def _logs():
return await manager.logs(follow=follow, tail=tail)
output = anyio.run(_logs)
console.print(output)
@cli.command("cleanup")
@click.option(
"--force",
is_flag=True,
help="Force cleanup even if state file doesn't exist"
)
def cleanup_cmd(force: bool):
"""Force cleanup of all Crawl4AI Docker resources.
Stops and removes all containers, networks, and optionally volumes.
Useful when server is stuck or state is corrupted.
Examples:
# Clean up everything
cnode cleanup
# Force cleanup (ignore state file)
cnode cleanup --force
"""
manager = ServerManager()
console.print(Panel(
f"[yellow]⚠️ Cleaning up Crawl4AI Docker resources[/yellow]\n\n"
f"This will stop and remove:\n"
f"- All Crawl4AI containers\n"
f"- Nginx load balancer\n"
f"- Redis instance\n"
f"- Docker networks\n"
f"- State files",
title="Cleanup",
border_style="yellow"
))
if not force and not Confirm.ask("[yellow]Continue with cleanup?[/yellow]"):
console.print("[yellow]Cancelled[/yellow]")
return
with console.status("[cyan]Cleaning up resources..."):
async def _cleanup():
return await manager.cleanup(force=force)
result = anyio.run(_cleanup)
if result["success"]:
console.print(Panel(
f"[green]✓ Cleanup completed successfully[/green]\n\n"
f"Removed: {result.get('removed', 0)} containers\n"
f"{result.get('message', 'All resources cleaned up')}",
title="Cleanup Complete",
border_style="green"
))
else:
console.print(Panel(
f"[yellow]⚠️ Partial cleanup[/yellow]\n\n"
f"{result.get('message', 'Some resources may still exist')}",
title="Cleanup Status",
border_style="yellow"
))
@cli.command("restart")
@click.option(
"--replicas", "-r",
type=int,
help="New replica count (optional)"
)
def restart_cmd(replicas: int):
"""Restart server (stop then start with same config).
Preserves existing configuration unless overridden with options.
Useful for applying image updates or recovering from errors.
Examples:
# Restart with same configuration
cnode restart
# Restart and change replica count
cnode restart --replicas 5
"""
manager = ServerManager()
# Get current state
async def _get_status():
return await manager.status()
current = anyio.run(_get_status)
if not current["running"]:
console.print("[yellow]No server is running. Use 'cnode start' instead.[/yellow]")
return
# Extract current config
current_replicas = current.get("replicas", 1)
current_port = current.get("port", 11235)
current_image = current.get("image", "unclecode/crawl4ai:latest")
current_mode = current.get("mode", "auto")
# Override with CLI args
new_replicas = replicas if replicas is not None else current_replicas
console.print(Panel(
f"[cyan]Restarting Crawl4AI Server[/cyan]\n\n"
f"Replicas: [yellow]{current_replicas}[/yellow] → [green]{new_replicas}[/green]\n"
f"Port: [yellow]{current_port}[/yellow]\n"
f"Mode: [yellow]{current_mode}[/yellow]",
title="Server Restart",
border_style="cyan"
))
# Stop current
with console.status("[cyan]Stopping current server..."):
async def _stop_server():
return await manager.stop(remove_volumes=False)
stop_result = anyio.run(_stop_server)
if not stop_result["success"]:
console.print(f"[red]Failed to stop server: {stop_result.get('error')}[/red]")
return
# Start new
with console.status("[cyan]Starting server..."):
async def _start_server():
return await manager.start(
replicas=new_replicas,
mode="auto",
port=current_port,
image=current_image
)
start_result = anyio.run(_start_server)
if start_result["success"]:
console.print(Panel(
f"[green]✓ Server restarted successfully![/green]\n\n"
f"URL: [bold]http://localhost:{current_port}[/bold]",
title="Restart Complete",
border_style="green"
))
else:
console.print(Panel(
f"[red]✗ Failed to restart server[/red]\n\n"
f"{start_result.get('error', 'Unknown error')}",
title="Error",
border_style="red"
))
def main():
"""Entry point for cnode CLI"""
cli()
if __name__ == "__main__":
main()
# Test comment

File diff suppressed because it is too large Load Diff

345
deploy/installer/README.md Normal file
View File

@@ -0,0 +1,345 @@
# Crawl4AI Node Manager (cnode) - Installation & Distribution
This directory contains the standalone `cnode` package and installation scripts for managing Crawl4AI Docker server instances.
## Overview
`cnode` is a fast, lightweight CLI tool for managing Crawl4AI Docker servers. It provides:
- One-command deployment with automatic scaling
- Single container for development (N=1)
- Docker Swarm for production with built-in load balancing (N>1)
- Docker Compose + Nginx as fallback (N>1)
## Directory Structure
```
deploy/installer/
├── README.md # This file
├── cnode_pkg/ # Standalone Python package
│ ├── __init__.py # Package marker
│ ├── cli.py # CLI interface (commands)
│ ├── server_manager.py # Docker orchestration logic
│ └── requirements.txt # Python dependencies
├── install-cnode.sh # Local installation script
├── deploy.sh # Remote installation script (for users)
└── releases/ # Release artifacts for distribution
```
## Installation
### For Users (Remote Installation)
Users can install `cnode` directly from the web:
```bash
# Install from GitHub/website
curl -sSL https://crawl4ai.com/install-cnode.sh | bash
# Or with wget
wget -qO- https://crawl4ai.com/install-cnode.sh | bash
```
### For Local Testing
Test the installation locally:
```bash
cd deploy/installer
./install-cnode.sh
```
## Package Contents
### `cnode_pkg/` - Python Package
This is a self-contained Python package with:
- **`cli.py`**: Click-based CLI with all commands (start, stop, status, scale, logs, cleanup, restart)
- **`server_manager.py`**: Core Docker orchestration logic
- **`requirements.txt`**: Dependencies (click, rich, anyio, pyyaml)
- **`__init__.py`**: Package initialization
### Installation Script
**`install-cnode.sh`** does the following:
1. Checks for Python 3.8+ and pip
2. Checks for Docker (warns if not found)
3. Installs Python dependencies
4. Copies `cnode_pkg/` to `/usr/local/lib/cnode/`
5. Creates wrapper script at `/usr/local/bin/cnode`
6. Verifies installation
### Wrapper Script
Created at `/usr/local/bin/cnode`:
```bash
#!/usr/bin/env bash
set -e
# Find Python
if command -v python3 &> /dev/null; then
PYTHON_CMD="python3"
elif command -v python &> /dev/null; then
PYTHON_CMD="python"
else
echo "Error: Python 3.8+ required" >&2
exit 1
fi
# Run cnode
export PYTHONPATH="/usr/local/lib/cnode:$PYTHONPATH"
exec $PYTHON_CMD -m cnode_pkg.cli "$@"
```
## Performance
**Blazing Fast Startup:**
- **~0.1 seconds** to launch
- 49x faster than compiled binary alternatives
- Minimal overhead, maximum responsiveness
## Requirements
### User Requirements
- Python 3.8 or higher
- pip (Python package manager)
- Docker (for running servers)
### Dependencies (Auto-installed)
- click >= 8.0.0 (CLI framework)
- rich >= 13.0.0 (Terminal formatting)
- anyio >= 3.0.0 (Async I/O)
- pyyaml >= 6.0.0 (YAML parsing)
## Usage
After installation:
```bash
# Quick start
cnode start # Single container on port 11235
cnode start --replicas 5 # 5-replica cluster
cnode status # Check server status
cnode logs -f # Follow logs
cnode scale 10 # Scale to 10 replicas
cnode stop # Stop server
# Get help
cnode --help
cnode start --help
```
## Development Workflow
### Making Changes
1. **Edit source code** in `deploy/docker/`:
```bash
vim deploy/docker/cnode_cli.py
vim deploy/docker/server_manager.py
```
2. **Update package** by copying to installer:
```bash
# Copy CLI
cp deploy/docker/cnode_cli.py deploy/installer/cnode_pkg/cli.py
# Fix imports (deploy.docker → cnode_pkg)
sed -i 's/from deploy\.docker\./from cnode_pkg./g' deploy/installer/cnode_pkg/cli.py
# Copy server manager
cp deploy/docker/server_manager.py deploy/installer/cnode_pkg/server_manager.py
```
3. **Test locally**:
```bash
cd deploy/installer
./install-cnode.sh
cnode --help
```
4. **Commit both**:
```bash
git add deploy/docker/cnode_cli.py
git add deploy/installer/cnode_pkg/cli.py
git commit -m "Update cnode: [description]"
```
### Creating a Release
1. **Tag the release**:
```bash
git tag -a v1.0.0 -m "Release v1.0.0"
git push origin v1.0.0
```
2. **Package for distribution**:
```bash
cd deploy/installer
tar -czf releases/cnode-v1.0.0.tar.gz cnode_pkg/ install-cnode.sh
```
3. **Create GitHub release**:
```bash
gh release create v1.0.0 \
releases/cnode-v1.0.0.tar.gz \
--title "cnode v1.0.0" \
--notes "Release notes here"
```
4. **Update deployment script** (if needed):
- Update `deploy.sh` with new version/URL
- Upload to hosting (e.g., `https://crawl4ai.com/install-cnode.sh`)
## Deployment
### Remote Installation Script
The `deploy.sh` script is meant to be hosted at a public URL for user installation:
```bash
# Upload to your server
scp deploy.sh user@crawl4ai.com:/var/www/html/install-cnode.sh
# Or use GitHub raw URL
https://raw.githubusercontent.com/unclecode/crawl4ai/main/deploy/installer/deploy.sh
```
Users can then install with:
```bash
curl -sSL https://crawl4ai.com/install-cnode.sh | bash
```
## Backward Compatibility
The main Crawl4AI CLI (`crwl`) includes a redirect for backward compatibility:
```bash
# These work identically:
crwl server start --replicas 3
cnode start --replicas 3
# All subcommands redirect:
crwl server status → cnode status
crwl server stop → cnode stop
crwl server scale 5 → cnode scale 5
crwl server logs -f → cnode logs -f
```
This ensures existing scripts continue working while users migrate to `cnode`.
## Uninstallation
To remove cnode:
```bash
# Remove command
sudo rm /usr/local/bin/cnode
# Remove package
sudo rm -rf /usr/local/lib/cnode
# (Optional) Uninstall dependencies
pip uninstall click rich anyio pyyaml
```
## Troubleshooting
### Python Not Found
```bash
# Install Python 3.8+
# macOS: brew install python3
# Ubuntu: sudo apt install python3 python3-pip
# RHEL/CentOS: sudo yum install python3 python3-pip
```
### Permission Denied
```bash
# Run installer with sudo
sudo ./install-cnode.sh
# Or change install location
INSTALL_DIR=$HOME/.local/bin ./install-cnode.sh
```
### Command Not Found After Install
```bash
# Add to PATH in ~/.bashrc or ~/.zshrc
export PATH="/usr/local/bin:$PATH"
# Reload shell
source ~/.bashrc # or source ~/.zshrc
```
### Dependencies Install Failed
```bash
# Install manually
pip install --user click rich anyio pyyaml
# Or with break-system-packages (if needed)
pip install --user --break-system-packages click rich anyio pyyaml
```
### Docker Not Running
```bash
# macOS: Start Docker Desktop
# Linux: sudo systemctl start docker
# Check Docker
docker --version
docker ps
```
## Architecture
### Component Flow
```
User runs: cnode start
/usr/local/bin/cnode (wrapper script)
Finds python3 executable
Sets PYTHONPATH=/usr/local/lib/cnode
python3 -m cnode_pkg.cli start
cli.py → start_cmd()
server_manager.py → ServerManager.start()
Docker orchestration (single/swarm/compose)
Server running!
```
### Why Python Wrapper vs Binary?
We chose a Python wrapper over compiled binaries (PyInstaller) because:
| Metric | Python Wrapper | PyInstaller Binary |
|--------|---------------|-------------------|
| Startup time | **0.1s** | 4.7s |
| Size | ~50KB wrapper | 8.8MB |
| Updates | Easy (just copy files) | Rebuild required |
| Dependencies | Python 3.8+ | None |
| Platform | Any with Python | OS-specific builds |
Since users running Crawl4AI already have Python, the wrapper is the clear winner.
## Support
For issues or questions:
- GitHub Issues: https://github.com/unclecode/crawl4ai/issues
- Documentation: https://docs.crawl4ai.com
- Discord: https://discord.gg/crawl4ai
## Version History
- **v1.0.0**: Initial release with Python wrapper approach
- Fast startup (~0.1s)
- Supports single container, Docker Swarm, and Compose modes
- Auto-scaling and load balancing
- Real-time monitoring and logs

View File

@@ -0,0 +1,5 @@
"""
Crawl4AI Node Manager (cnode) - Docker server orchestration CLI
"""
__version__ = "1.0.0"

View File

@@ -0,0 +1,492 @@
"""
Crawl4AI Server CLI Commands
Provides `cnode` command group for Docker orchestration.
"""
import click
import anyio
from rich.console import Console
from rich.table import Table
from rich.panel import Panel
from rich.prompt import Confirm
from cnode_pkg.server_manager import ServerManager
console = Console()
@click.group()
def cli():
"""Manage Crawl4AI Docker server instances
\b
One-command deployment with automatic scaling:
• Single container for development (N=1)
• Docker Swarm for production with built-in load balancing (N>1)
• Docker Compose + Nginx as fallback (N>1)
\b
Examples:
cnode start # Single container on port 11235
cnode start --replicas 3 # Auto-detect Swarm or Compose
cnode start -r 5 --port 8080 # 5 replicas on custom port
cnode status # Check current deployment
cnode scale 10 # Scale to 10 replicas
cnode stop # Stop and cleanup
"""
pass
@cli.command("start")
@click.option(
"--replicas", "-r",
type=int,
default=1,
help="Number of container replicas (default: 1)"
)
@click.option(
"--mode",
type=click.Choice(["auto", "single", "swarm", "compose"]),
default="auto",
help="Deployment mode (default: auto-detect)"
)
@click.option(
"--port", "-p",
type=int,
default=11235,
help="External port to expose (default: 11235)"
)
@click.option(
"--env-file",
type=click.Path(exists=True),
help="Path to environment file"
)
@click.option(
"--image",
default="unclecode/crawl4ai:latest",
help="Docker image to use (default: unclecode/crawl4ai:latest)"
)
def start_cmd(replicas: int, mode: str, port: int, env_file: str, image: str):
"""Start Crawl4AI server with automatic orchestration.
Deployment modes:
- auto: Automatically choose best mode (default)
- single: Single container (N=1 only)
- swarm: Docker Swarm with built-in load balancing
- compose: Docker Compose + Nginx reverse proxy
The server will:
1. Check if Docker is running
2. Validate port availability
3. Pull image if needed
4. Start container(s) with health checks
5. Save state for management
Examples:
# Development: single container
cnode start
# Production: 5 replicas with Swarm
cnode start --replicas 5
# Custom configuration
cnode start -r 3 --port 8080 --env-file .env.prod
"""
manager = ServerManager()
console.print(Panel(
f"[cyan]Starting Crawl4AI Server[/cyan]\n\n"
f"Replicas: [yellow]{replicas}[/yellow]\n"
f"Mode: [yellow]{mode}[/yellow]\n"
f"Port: [yellow]{port}[/yellow]\n"
f"Image: [yellow]{image}[/yellow]",
title="Server Start",
border_style="cyan"
))
with console.status("[cyan]Starting server..."):
async def _start():
return await manager.start(
replicas=replicas,
mode=mode,
port=port,
env_file=env_file,
image=image
)
result = anyio.run(_start)
if result["success"]:
console.print(Panel(
f"[green]✓ Server started successfully![/green]\n\n"
f"Mode: [cyan]{result.get('state_data', {}).get('mode', mode)}[/cyan]\n"
f"URL: [bold]http://localhost:{port}[/bold]\n"
f"Health: [bold]http://localhost:{port}/health[/bold]\n"
f"Monitor: [bold]http://localhost:{port}/monitor[/bold]",
title="Server Running",
border_style="green"
))
else:
error_msg = result.get("error", result.get("message", "Unknown error"))
console.print(Panel(
f"[red]✗ Failed to start server[/red]\n\n"
f"{error_msg}",
title="Error",
border_style="red"
))
if "already running" in error_msg.lower():
console.print("\n[yellow]Hint: Use 'cnode status' to check current deployment[/yellow]")
console.print("[yellow] Use 'cnode stop' to stop existing server[/yellow]")
@cli.command("status")
def status_cmd():
"""Show current server status and deployment info.
Displays:
- Running state (up/down)
- Deployment mode (single/swarm/compose)
- Number of replicas
- Port mapping
- Uptime
- Image version
Example:
cnode status
"""
manager = ServerManager()
async def _status():
return await manager.status()
result = anyio.run(_status)
if result["running"]:
table = Table(title="Crawl4AI Server Status", border_style="green")
table.add_column("Property", style="cyan")
table.add_column("Value", style="green")
table.add_row("Status", "🟢 Running")
table.add_row("Mode", result["mode"])
table.add_row("Replicas", str(result.get("replicas", 1)))
table.add_row("Port", str(result.get("port", 11235)))
table.add_row("Image", result.get("image", "unknown"))
table.add_row("Uptime", result.get("uptime", "unknown"))
table.add_row("Started", result.get("started_at", "unknown"))
console.print(table)
console.print(f"\n[green]✓ Server is healthy[/green]")
console.print(f"[dim]Access: http://localhost:{result.get('port', 11235)}[/dim]")
else:
console.print(Panel(
f"[yellow]No server is currently running[/yellow]\n\n"
f"Use 'cnode start' to launch a server",
title="Server Status",
border_style="yellow"
))
@cli.command("stop")
@click.option(
"--remove-volumes",
is_flag=True,
help="Remove associated volumes (WARNING: deletes data)"
)
def stop_cmd(remove_volumes: bool):
"""Stop running Crawl4AI server and cleanup resources.
This will:
1. Stop all running containers/services
2. Remove containers
3. Optionally remove volumes (--remove-volumes)
4. Clean up state files
WARNING: Use --remove-volumes with caution as it will delete
persistent data including Redis databases and logs.
Examples:
# Stop server, keep volumes
cnode stop
# Stop and remove all data
cnode stop --remove-volumes
"""
manager = ServerManager()
# Confirm if removing volumes
if remove_volumes:
if not Confirm.ask(
"[red]⚠️ This will delete all server data including Redis databases. Continue?[/red]"
):
console.print("[yellow]Cancelled[/yellow]")
return
with console.status("[cyan]Stopping server..."):
async def _stop():
return await manager.stop(remove_volumes=remove_volumes)
result = anyio.run(_stop)
if result["success"]:
console.print(Panel(
f"[green]✓ Server stopped successfully[/green]\n\n"
f"{result.get('message', 'All resources cleaned up')}",
title="Server Stopped",
border_style="green"
))
else:
console.print(Panel(
f"[red]✗ Error stopping server[/red]\n\n"
f"{result.get('error', result.get('message', 'Unknown error'))}",
title="Error",
border_style="red"
))
@cli.command("scale")
@click.argument("replicas", type=int)
def scale_cmd(replicas: int):
"""Scale server to specified number of replicas.
Only works with Swarm or Compose modes. Single container
mode cannot be scaled (must stop and restart with --replicas).
Scaling is live and does not require downtime. The load
balancer will automatically distribute traffic to new replicas.
Examples:
# Scale up to 10 replicas
cnode scale 10
# Scale down to 2 replicas
cnode scale 2
# Scale to 1 (minimum)
cnode scale 1
"""
if replicas < 1:
console.print("[red]Error: Replicas must be at least 1[/red]")
return
manager = ServerManager()
with console.status(f"[cyan]Scaling to {replicas} replicas..."):
async def _scale():
return await manager.scale(replicas=replicas)
result = anyio.run(_scale)
if result["success"]:
console.print(Panel(
f"[green]✓ Scaled successfully[/green]\n\n"
f"New replica count: [bold]{replicas}[/bold]\n"
f"Mode: [cyan]{result.get('mode')}[/cyan]",
title="Scaling Complete",
border_style="green"
))
else:
error_msg = result.get("error", result.get("message", "Unknown error"))
console.print(Panel(
f"[red]✗ Scaling failed[/red]\n\n"
f"{error_msg}",
title="Error",
border_style="red"
))
if "single container" in error_msg.lower():
console.print("\n[yellow]Hint: For single container mode:[/yellow]")
console.print("[yellow] 1. cnode stop[/yellow]")
console.print(f"[yellow] 2. cnode start --replicas {replicas}[/yellow]")
@cli.command("logs")
@click.option(
"--follow", "-f",
is_flag=True,
help="Follow log output (like tail -f)"
)
@click.option(
"--tail",
type=int,
default=100,
help="Number of lines to show (default: 100)"
)
def logs_cmd(follow: bool, tail: int):
"""View server logs.
Shows logs from running containers/services. Use --follow
to stream logs in real-time.
Examples:
# Show last 100 lines
cnode logs
# Show last 500 lines
cnode logs --tail 500
# Follow logs in real-time
cnode logs --follow
# Combine options
cnode logs -f --tail 50
"""
manager = ServerManager()
async def _logs():
return await manager.logs(follow=follow, tail=tail)
output = anyio.run(_logs)
console.print(output)
@cli.command("cleanup")
@click.option(
"--force",
is_flag=True,
help="Force cleanup even if state file doesn't exist"
)
def cleanup_cmd(force: bool):
"""Force cleanup of all Crawl4AI Docker resources.
Stops and removes all containers, networks, and optionally volumes.
Useful when server is stuck or state is corrupted.
Examples:
# Clean up everything
cnode cleanup
# Force cleanup (ignore state file)
cnode cleanup --force
"""
manager = ServerManager()
console.print(Panel(
f"[yellow]⚠️ Cleaning up Crawl4AI Docker resources[/yellow]\n\n"
f"This will stop and remove:\n"
f"- All Crawl4AI containers\n"
f"- Nginx load balancer\n"
f"- Redis instance\n"
f"- Docker networks\n"
f"- State files",
title="Cleanup",
border_style="yellow"
))
if not force and not Confirm.ask("[yellow]Continue with cleanup?[/yellow]"):
console.print("[yellow]Cancelled[/yellow]")
return
with console.status("[cyan]Cleaning up resources..."):
async def _cleanup():
return await manager.cleanup(force=force)
result = anyio.run(_cleanup)
if result["success"]:
console.print(Panel(
f"[green]✓ Cleanup completed successfully[/green]\n\n"
f"Removed: {result.get('removed', 0)} containers\n"
f"{result.get('message', 'All resources cleaned up')}",
title="Cleanup Complete",
border_style="green"
))
else:
console.print(Panel(
f"[yellow]⚠️ Partial cleanup[/yellow]\n\n"
f"{result.get('message', 'Some resources may still exist')}",
title="Cleanup Status",
border_style="yellow"
))
@cli.command("restart")
@click.option(
"--replicas", "-r",
type=int,
help="New replica count (optional)"
)
def restart_cmd(replicas: int):
"""Restart server (stop then start with same config).
Preserves existing configuration unless overridden with options.
Useful for applying image updates or recovering from errors.
Examples:
# Restart with same configuration
cnode restart
# Restart and change replica count
cnode restart --replicas 5
"""
manager = ServerManager()
# Get current state
async def _get_status():
return await manager.status()
current = anyio.run(_get_status)
if not current["running"]:
console.print("[yellow]No server is running. Use 'cnode start' instead.[/yellow]")
return
# Extract current config
current_replicas = current.get("replicas", 1)
current_port = current.get("port", 11235)
current_image = current.get("image", "unclecode/crawl4ai:latest")
current_mode = current.get("mode", "auto")
# Override with CLI args
new_replicas = replicas if replicas is not None else current_replicas
console.print(Panel(
f"[cyan]Restarting Crawl4AI Server[/cyan]\n\n"
f"Replicas: [yellow]{current_replicas}[/yellow] → [green]{new_replicas}[/green]\n"
f"Port: [yellow]{current_port}[/yellow]\n"
f"Mode: [yellow]{current_mode}[/yellow]",
title="Server Restart",
border_style="cyan"
))
# Stop current
with console.status("[cyan]Stopping current server..."):
async def _stop_server():
return await manager.stop(remove_volumes=False)
stop_result = anyio.run(_stop_server)
if not stop_result["success"]:
console.print(f"[red]Failed to stop server: {stop_result.get('error')}[/red]")
return
# Start new
with console.status("[cyan]Starting server..."):
async def _start_server():
return await manager.start(
replicas=new_replicas,
mode="auto",
port=current_port,
image=current_image
)
start_result = anyio.run(_start_server)
if start_result["success"]:
console.print(Panel(
f"[green]✓ Server restarted successfully![/green]\n\n"
f"URL: [bold]http://localhost:{current_port}[/bold]",
title="Restart Complete",
border_style="green"
))
else:
console.print(Panel(
f"[red]✗ Failed to restart server[/red]\n\n"
f"{start_result.get('error', 'Unknown error')}",
title="Error",
border_style="red"
))
def main():
"""Entry point for cnode CLI"""
cli()
if __name__ == "__main__":
main()
# Test comment

View File

@@ -0,0 +1,4 @@
click>=8.0.0
rich>=13.0.0
anyio>=3.0.0
pyyaml>=6.0.0

File diff suppressed because it is too large Load Diff

217
deploy/installer/deploy.sh Executable file
View File

@@ -0,0 +1,217 @@
#!/bin/bash
# Crawl4AI Node Manager (cnode) Installation Script
# Usage: curl -sSL https://crawl4ai.com/deploy.sh | bash
# Or: wget -qO- https://crawl4ai.com/deploy.sh | bash
set -e
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Configuration
INSTALL_DIR="${INSTALL_DIR:-/usr/local/bin}"
BINARY_NAME="cnode"
GITHUB_REPO="unclecode/crawl4ai"
RELEASE_TAG="${CNODE_VERSION:-latest}"
echo -e "${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Crawl4AI Node Manager (cnode) Installation Script ║${NC}"
echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}\n"
# Detect OS and architecture
detect_platform() {
OS="$(uname -s)"
ARCH="$(uname -m)"
case "$OS" in
Linux*)
OS_TYPE="linux"
;;
Darwin*)
OS_TYPE="macos"
;;
*)
echo -e "${RED}Error: Unsupported operating system: $OS${NC}"
exit 1
;;
esac
case "$ARCH" in
x86_64|amd64)
ARCH_TYPE="amd64"
;;
aarch64|arm64)
ARCH_TYPE="arm64"
;;
*)
echo -e "${RED}Error: Unsupported architecture: $ARCH${NC}"
exit 1
;;
esac
echo -e "${BLUE}Detected platform: ${YELLOW}$OS_TYPE-$ARCH_TYPE${NC}"
}
# Check if Docker is installed
check_docker() {
if ! command -v docker &> /dev/null; then
echo -e "${YELLOW}⚠️ Docker not found. cnode requires Docker to manage server instances.${NC}"
echo -e "${YELLOW}Install Docker from: https://docs.docker.com/get-docker/${NC}\n"
read -p "Continue installation anyway? (y/N) " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
exit 1
fi
else
echo -e "${GREEN}✓ Docker is installed${NC}"
fi
}
# Check write permissions
check_permissions() {
if [ ! -w "$INSTALL_DIR" ]; then
echo -e "${YELLOW}⚠️ No write permission for $INSTALL_DIR${NC}"
echo -e "${YELLOW}The script will attempt to use sudo for installation.${NC}\n"
USE_SUDO="sudo"
else
USE_SUDO=""
fi
}
# Download binary
download_binary() {
BINARY_URL="https://github.com/$GITHUB_REPO/releases/download/$RELEASE_TAG/cnode-$OS_TYPE-$ARCH_TYPE"
echo -e "${BLUE}Downloading cnode from GitHub...${NC}"
echo -e "${YELLOW}URL: $BINARY_URL${NC}\n"
# Create temp directory
TMP_DIR="$(mktemp -d)"
TMP_FILE="$TMP_DIR/$BINARY_NAME"
# Download with curl or wget
if command -v curl &> /dev/null; then
if ! curl -fSL "$BINARY_URL" -o "$TMP_FILE"; then
echo -e "${RED}Error: Failed to download binary${NC}"
echo -e "${YELLOW}URL: $BINARY_URL${NC}"
rm -rf "$TMP_DIR"
exit 1
fi
elif command -v wget &> /dev/null; then
if ! wget -q "$BINARY_URL" -O "$TMP_FILE"; then
echo -e "${RED}Error: Failed to download binary${NC}"
echo -e "${YELLOW}URL: $BINARY_URL${NC}"
rm -rf "$TMP_DIR"
exit 1
fi
else
echo -e "${RED}Error: Neither curl nor wget found${NC}"
echo -e "${YELLOW}Please install curl or wget and try again${NC}"
rm -rf "$TMP_DIR"
exit 1
fi
# Make executable
chmod +x "$TMP_FILE"
echo "$TMP_FILE"
}
# Install binary
install_binary() {
local tmp_file="$1"
local install_path="$INSTALL_DIR/$BINARY_NAME"
echo -e "\n${BLUE}Installing cnode to $install_path...${NC}"
# Check if already installed
if [ -f "$install_path" ]; then
echo -e "${YELLOW}⚠️ cnode is already installed${NC}"
read -p "Overwrite existing installation? (y/N) " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
echo -e "${YELLOW}Installation cancelled${NC}"
rm -rf "$(dirname "$tmp_file")"
exit 0
fi
fi
# Install
if ! $USE_SUDO mv "$tmp_file" "$install_path"; then
echo -e "${RED}Error: Failed to install binary${NC}"
rm -rf "$(dirname "$tmp_file")"
exit 1
fi
# Cleanup temp directory
rm -rf "$(dirname "$tmp_file")"
echo -e "${GREEN}✓ Installation successful${NC}"
}
# Verify installation
verify_installation() {
echo -e "\n${BLUE}Verifying installation...${NC}"
if ! command -v $BINARY_NAME &> /dev/null; then
echo -e "${RED}Error: $BINARY_NAME not found in PATH${NC}"
echo -e "${YELLOW}You may need to add $INSTALL_DIR to your PATH${NC}"
echo -e "${YELLOW}Add this to your ~/.bashrc or ~/.zshrc:${NC}"
echo -e "${YELLOW}export PATH=\"$INSTALL_DIR:\$PATH\"${NC}\n"
exit 1
fi
# Test version
if $BINARY_NAME --help &> /dev/null; then
echo -e "${GREEN}$BINARY_NAME is working correctly${NC}"
else
echo -e "${RED}Error: $BINARY_NAME failed to execute${NC}"
exit 1
fi
}
# Show completion message
show_completion() {
local version
version=$($BINARY_NAME --help | head -1 || echo "unknown")
echo -e "\n${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Installation Complete! ║${NC}"
echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}\n"
echo -e "${BLUE}cnode is now installed and ready to use!${NC}\n"
echo -e "${YELLOW}Quick Start:${NC}"
echo -e " ${GREEN}cnode start${NC} # Start single server instance"
echo -e " ${GREEN}cnode start --replicas 5${NC} # Start 5-replica cluster"
echo -e " ${GREEN}cnode status${NC} # Check server status"
echo -e " ${GREEN}cnode logs -f${NC} # Follow server logs"
echo -e " ${GREEN}cnode scale 10${NC} # Scale to 10 replicas"
echo -e " ${GREEN}cnode stop${NC} # Stop server"
echo -e "\n${YELLOW}For more information:${NC}"
echo -e " ${BLUE}cnode --help${NC}"
echo -e " ${BLUE}https://github.com/$GITHUB_REPO${NC}\n"
}
# Main installation flow
main() {
detect_platform
check_docker
check_permissions
# Download and install
TMP_FILE=$(download_binary)
install_binary "$TMP_FILE"
# Verify
verify_installation
show_completion
}
# Run installation
main

142
deploy/installer/install-cnode.sh Executable file
View File

@@ -0,0 +1,142 @@
#!/bin/bash
# Crawl4AI Node Manager (cnode) - Fast Installation Script
# This installs cnode as a Python package with a wrapper script
set -e
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'
echo -e "${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Crawl4AI Node Manager (cnode) - Fast Installer ║${NC}"
echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}\n"
# Configuration
INSTALL_DIR="${INSTALL_DIR:-/usr/local/bin}"
LIB_DIR="${LIB_DIR:-/usr/local/lib/cnode}"
# Check Python
echo -e "${BLUE}Checking Python installation...${NC}"
if command -v python3 &> /dev/null; then
PYTHON_CMD="python3"
elif command -v python &> /dev/null; then
PYTHON_CMD="python"
else
echo -e "${RED}Error: Python 3.8+ is required but not found${NC}"
echo -e "${YELLOW}Install from: https://www.python.org/downloads/${NC}"
exit 1
fi
# Check Python version
PYTHON_VERSION=$($PYTHON_CMD -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
echo -e "${GREEN}✓ Found Python $PYTHON_VERSION${NC}"
if [ "$(printf '%s\n' "3.8" "$PYTHON_VERSION" | sort -V | head -n1)" != "3.8" ]; then
echo -e "${RED}Error: Python 3.8+ required, found $PYTHON_VERSION${NC}"
exit 1
fi
# Check pip
if ! $PYTHON_CMD -m pip --version &> /dev/null; then
echo -e "${RED}Error: pip is required${NC}"
echo -e "${YELLOW}Install pip: $PYTHON_CMD -m ensurepip${NC}"
exit 1
fi
echo -e "${GREEN}✓ pip is available${NC}"
# Check Docker
echo -e "\n${BLUE}Checking Docker...${NC}"
if ! command -v docker &> /dev/null; then
echo -e "${YELLOW}⚠️ Docker not found (required for running servers)${NC}"
echo -e "${YELLOW}Install from: https://docs.docker.com/get-docker/${NC}\n"
else
echo -e "${GREEN}✓ Docker is installed${NC}"
fi
# Check permissions
USE_SUDO=""
if [ ! -w "$INSTALL_DIR" ] || [ ! -w "/usr/local" ]; then
echo -e "\n${YELLOW}⚠️ Root permission required for installation${NC}"
USE_SUDO="sudo"
fi
# Get script directory
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Install Python dependencies
echo -e "\n${BLUE}Installing Python dependencies...${NC}"
$PYTHON_CMD -m pip install --quiet --user -r "$SCRIPT_DIR/cnode_pkg/requirements.txt" 2>/dev/null || \
$PYTHON_CMD -m pip install --quiet --user --break-system-packages -r "$SCRIPT_DIR/cnode_pkg/requirements.txt" 2>/dev/null || {
echo -e "${YELLOW}⚠️ Could not install dependencies with pip${NC}"
echo -e "${YELLOW}Trying to continue anyway (dependencies may already be installed)${NC}"
}
echo -e "${GREEN}✓ Dependencies check complete${NC}"
# Create lib directory
echo -e "\n${BLUE}Installing cnode package...${NC}"
$USE_SUDO mkdir -p "$LIB_DIR"
$USE_SUDO cp -r "$SCRIPT_DIR/cnode_pkg" "$LIB_DIR/"
echo -e "${GREEN}✓ Package installed to $LIB_DIR${NC}"
# Create wrapper script
echo -e "\n${BLUE}Creating cnode command...${NC}"
$USE_SUDO tee "$INSTALL_DIR/cnode" > /dev/null << 'EOF'
#!/usr/bin/env bash
# Crawl4AI Node Manager (cnode) wrapper
set -e
# Find Python
if command -v python3 &> /dev/null; then
PYTHON_CMD="python3"
elif command -v python &> /dev/null; then
PYTHON_CMD="python"
else
echo "Error: Python 3.8+ required" >&2
exit 1
fi
# Add cnode to Python path and run
export PYTHONPATH="/usr/local/lib/cnode:$PYTHONPATH"
exec $PYTHON_CMD -m cnode_pkg.cli "$@"
EOF
$USE_SUDO chmod +x "$INSTALL_DIR/cnode"
echo -e "${GREEN}✓ cnode command created${NC}"
# Verify installation
echo -e "\n${BLUE}Verifying installation...${NC}"
if ! command -v cnode &> /dev/null; then
echo -e "${RED}Error: cnode not found in PATH${NC}"
echo -e "${YELLOW}Add $INSTALL_DIR to your PATH${NC}"
exit 1
fi
if ! cnode --help &> /dev/null; then
echo -e "${RED}Error: cnode command failed${NC}"
exit 1
fi
echo -e "${GREEN}✓ Installation verified${NC}"
# Success message
echo -e "\n${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Installation Complete! ║${NC}"
echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}\n"
echo -e "${BLUE}cnode is now installed and ready!${NC}\n"
echo -e "${YELLOW}Quick Start:${NC}"
echo -e " ${GREEN}cnode start${NC} # Start single server"
echo -e " ${GREEN}cnode start --replicas 5${NC} # Start 5-replica cluster"
echo -e " ${GREEN}cnode status${NC} # Check status"
echo -e " ${GREEN}cnode logs -f${NC} # Follow logs"
echo -e " ${GREEN}cnode stop${NC} # Stop server"
echo -e "\n${YELLOW}More help:${NC}"
echo -e " ${BLUE}cnode --help${NC}\n"

33
deploy/installer/sync-cnode.sh Executable file
View File

@@ -0,0 +1,33 @@
#!/bin/bash
# Sync cnode source code to installer package
# Run this before committing changes to cnode
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SOURCE_DIR="$SCRIPT_DIR/../docker"
PKG_DIR="$SCRIPT_DIR/cnode_pkg"
echo "🔄 Syncing cnode source to package..."
# Copy CLI
echo " → Copying cnode_cli.py to cli.py"
cp "$SOURCE_DIR/cnode_cli.py" "$PKG_DIR/cli.py"
# Fix imports
echo " → Fixing imports (deploy.docker → cnode_pkg)"
sed -i '' 's/from deploy\.docker\./from cnode_pkg./g' "$PKG_DIR/cli.py"
# Copy server manager
echo " → Copying server_manager.py"
cp "$SOURCE_DIR/server_manager.py" "$PKG_DIR/server_manager.py"
echo "✅ Sync complete!"
echo ""
echo "Files updated:"
echo " • deploy/installer/cnode_pkg/cli.py"
echo " • deploy/installer/cnode_pkg/server_manager.py"
echo ""
echo "Next steps:"
echo " 1. Test: cd deploy/installer && ./install-cnode.sh"
echo " 2. Commit both source and package files"

24
setup-hooks.sh Executable file
View File

@@ -0,0 +1,24 @@
#!/bin/bash
# Setup Git hooks for cnode auto-sync
# Run this once after cloning the repo: ./setup-hooks.sh
set -e
echo "🔧 Setting up Git hooks..."
# Configure Git to use .githooks directory
git config core.hooksPath .githooks
echo "✅ Git hooks configured!"
echo ""
echo "Hooks installed:"
echo " • pre-commit: Auto-syncs cnode source → package when committing"
echo ""
echo "What this means:"
echo " ✅ Edit deploy/docker/cnode_cli.py"
echo " ✅ Run: git add deploy/docker/cnode_cli.py"
echo " ✅ Run: git commit -m \"update cnode\""
echo " ✅ Hook automatically syncs to deploy/installer/cnode_pkg/"
echo " ✅ Synced files are auto-staged in the same commit"
echo ""
echo "You're all set! 🚀"

View File

@@ -65,14 +65,14 @@ class TestCLIBasics:
assert 'Crawl4AI CLI' in result.output assert 'Crawl4AI CLI' in result.output
def test_examples(self, runner): def test_examples(self, runner):
result = runner.invoke(cli, ['--example']) result = runner.invoke(cli, ['examples'])
assert result.exit_code == 0 assert result.exit_code == 0
assert 'Examples' in result.output assert 'Examples' in result.output
def test_missing_url(self, runner): def test_missing_url(self, runner):
result = runner.invoke(cli) result = runner.invoke(cli, ['crawl'])
assert result.exit_code != 0 assert result.exit_code != 0
assert 'URL argument is required' in result.output assert ('Missing argument' in result.output or 'required' in result.output.lower())
class TestConfigParsing: class TestConfigParsing:
def test_parse_key_values_basic(self): def test_parse_key_values_basic(self):
@@ -101,18 +101,19 @@ class TestConfigLoading:
class TestLLMConfig: class TestLLMConfig:
def test_llm_config_creation(self, temp_config_dir, runner): def test_llm_config_creation(self, temp_config_dir, runner):
def input_simulation(inputs): def input_simulation(inputs):
return runner.invoke(cli, ['https://example.com', '-q', 'test question'], return runner.invoke(cli, ['crawl', 'https://example.com', '-q', 'test question'],
input='\n'.join(inputs)) input='\n'.join(inputs))
class TestCrawlingFeatures: class TestCrawlingFeatures:
def test_basic_crawl(self, runner): def test_basic_crawl(self, runner):
result = runner.invoke(cli, ['https://example.com']) result = runner.invoke(cli, ['crawl', 'https://example.com'])
assert result.exit_code == 0 assert result.exit_code == 0
class TestErrorHandling: class TestErrorHandling:
def test_invalid_config_file(self, runner): def test_invalid_config_file(self, runner):
result = runner.invoke(cli, [ result = runner.invoke(cli, [
'crawl',
'https://example.com', 'https://example.com',
'--browser-config', 'nonexistent.yml' '--browser-config', 'nonexistent.yml'
]) ])
@@ -124,6 +125,7 @@ class TestErrorHandling:
f.write('invalid json') f.write('invalid json')
result = runner.invoke(cli, [ result = runner.invoke(cli, [
'crawl',
'https://example.com', 'https://example.com',
'--schema', str(invalid_schema) '--schema', str(invalid_schema)
]) ])