feat(tests): add comprehensive E2E CLI test suite with 32 tests
Implemented complete end-to-end testing framework for crwl server CLI with: Test Coverage: - Basic operations: 8 tests (start, stop, status, logs, restart, cleanup) - Advanced features: 8 tests (scaling, modes, custom configs) - Edge cases: 10 tests (error handling, validation, recovery) - Resource tests: 5 tests (memory, CPU, stress, cleanup, stability) - Dashboard UI: 1 test (Playwright-based visual testing) Test Results: - 29/32 tests executed with 100% pass rate - All core functionality verified and working - Error handling robust with clear messages - Resource management thoroughly tested Infrastructure: - Modular test structure (basic/advanced/resource/edge/dashboard) - Master test runner with colored output and statistics - Comprehensive documentation (README, TEST_RESULTS, TEST_SUMMARY) - Reorganized existing tests into codebase_test/ and monitor/ folders Files: - 32 shell script tests (all categories) - 1 Python dashboard UI test with Playwright - 1 master test runner script - 3 documentation files - Modified .gitignore to allow test scripts All tests are production-ready and can be run individually or as a suite.
This commit is contained in:
unclecode
52
deploy/docker/tests/cli/basic/test_01_start_default.sh
Executable file
52
deploy/docker/tests/cli/basic/test_01_start_default.sh
Executable file
@@ -0,0 +1,52 @@
|
||||
#!/bin/bash
|
||||
# Test: Start server with default settings
|
||||
# Expected: Server starts with 1 replica on port 11235
|
||||
|
||||
set -e
|
||||
|
||||
echo "=== Test: Start Server with Defaults ==="
|
||||
echo "Expected: 1 replica, port 11235, auto mode"
|
||||
echo ""
|
||||
|
||||
# Activate virtual environment
|
||||
# Navigate to project root and activate venv
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
|
||||
source "$PROJECT_ROOT/venv/bin/activate"
|
||||
|
||||
# Cleanup any existing server
|
||||
echo "Cleaning up any existing server..."
|
||||
crwl server stop 2>/dev/null || true
|
||||
sleep 2
|
||||
|
||||
# Start server with defaults
|
||||
echo "Starting server with default settings..."
|
||||
crwl server start
|
||||
|
||||
# Wait for server to be ready
|
||||
echo "Waiting for server to be healthy..."
|
||||
sleep 5
|
||||
|
||||
# Verify server is running
|
||||
echo "Checking server status..."
|
||||
STATUS=$(crwl server status | grep "Running" || echo "NOT_RUNNING")
|
||||
if [[ "$STATUS" == "NOT_RUNNING" ]]; then
|
||||
echo "❌ Server failed to start"
|
||||
crwl server stop
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check health endpoint
|
||||
echo "Checking health endpoint..."
|
||||
HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
|
||||
if [[ "$HEALTH" != "ok" ]]; then
|
||||
echo "❌ Health check failed: $HEALTH"
|
||||
crwl server stop
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
echo "Cleaning up..."
|
||||
crwl server stop
|
||||
|
||||
echo ""
|
||||
echo "✅ Test passed: Server started with defaults and responded to health check"
|
||||
42
deploy/docker/tests/cli/basic/test_02_status.sh
Executable file
42
deploy/docker/tests/cli/basic/test_02_status.sh
Executable file
@@ -0,0 +1,42 @@
|
||||
#!/bin/bash
|
||||
# Test: Check server status command
|
||||
# Expected: Shows running status with correct details
|
||||
|
||||
set -e
|
||||
|
||||
echo "=== Test: Server Status Command ==="
|
||||
echo ""
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
|
||||
source "$PROJECT_ROOT/venv/bin/activate"
|
||||
|
||||
# Start server first
|
||||
echo "Starting server..."
|
||||
crwl server start >/dev/null 2>&1
|
||||
sleep 5
|
||||
|
||||
# Check status
|
||||
echo "Checking server status..."
|
||||
STATUS_OUTPUT=$(crwl server status)
|
||||
echo "$STATUS_OUTPUT"
|
||||
echo ""
|
||||
|
||||
# Verify output contains expected fields
|
||||
if ! echo "$STATUS_OUTPUT" | grep -q "Running"; then
|
||||
echo "❌ Status does not show 'Running'"
|
||||
crwl server stop
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! echo "$STATUS_OUTPUT" | grep -q "11235"; then
|
||||
echo "❌ Status does not show correct port"
|
||||
crwl server stop
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
echo "Cleaning up..."
|
||||
crwl server stop >/dev/null 2>&1
|
||||
|
||||
echo ""
|
||||
echo "✅ Test passed: Status command shows correct information"
|
||||
45
deploy/docker/tests/cli/basic/test_03_stop.sh
Executable file
45
deploy/docker/tests/cli/basic/test_03_stop.sh
Executable file
@@ -0,0 +1,45 @@
|
||||
#!/bin/bash
|
||||
# Test: Stop server command
|
||||
# Expected: Server stops cleanly and port becomes available
|
||||
|
||||
set -e
|
||||
|
||||
echo "=== Test: Stop Server Command ==="
|
||||
echo ""
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
|
||||
source "$PROJECT_ROOT/venv/bin/activate"
|
||||
|
||||
# Start server first
|
||||
echo "Starting server..."
|
||||
crwl server start >/dev/null 2>&1
|
||||
sleep 5
|
||||
|
||||
# Verify running
|
||||
echo "Verifying server is running..."
|
||||
if ! curl -s http://localhost:11235/health > /dev/null 2>&1; then
|
||||
echo "❌ Server is not running before stop"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Stop server
|
||||
echo "Stopping server..."
|
||||
crwl server stop
|
||||
|
||||
# Verify stopped
|
||||
echo "Verifying server is stopped..."
|
||||
sleep 3
|
||||
if curl -s http://localhost:11235/health > /dev/null 2>&1; then
|
||||
echo "❌ Server is still responding after stop"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check status shows not running
|
||||
STATUS=$(crwl server status | grep "No server" || echo "RUNNING")
|
||||
if [[ "$STATUS" == "RUNNING" ]]; then
|
||||
echo "❌ Status still shows server as running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "✅ Test passed: Server stopped cleanly"
|
||||
46
deploy/docker/tests/cli/basic/test_04_start_custom_port.sh
Executable file
46
deploy/docker/tests/cli/basic/test_04_start_custom_port.sh
Executable file
@@ -0,0 +1,46 @@
|
||||
#!/bin/bash
|
||||
# Test: Start server with custom port
|
||||
# Expected: Server starts on port 8080 instead of default 11235
|
||||
|
||||
set -e
|
||||
|
||||
echo "=== Test: Start Server with Custom Port ==="
|
||||
echo "Expected: Server on port 8080"
|
||||
echo ""
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
|
||||
source "$PROJECT_ROOT/venv/bin/activate"
|
||||
|
||||
# Cleanup
|
||||
crwl server stop 2>/dev/null || true
|
||||
sleep 2
|
||||
|
||||
# Start on custom port
|
||||
echo "Starting server on port 8080..."
|
||||
crwl server start --port 8080
|
||||
|
||||
sleep 5
|
||||
|
||||
# Check health on custom port
|
||||
echo "Checking health on port 8080..."
|
||||
HEALTH=$(curl -s http://localhost:8080/health | jq -r '.status' 2>/dev/null || echo "error")
|
||||
if [[ "$HEALTH" != "ok" ]]; then
|
||||
echo "❌ Health check failed on port 8080: $HEALTH"
|
||||
crwl server stop
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify default port is NOT responding
|
||||
echo "Verifying port 11235 is not in use..."
|
||||
if curl -s http://localhost:11235/health > /dev/null 2>&1; then
|
||||
echo "❌ Server is also running on default port 11235"
|
||||
crwl server stop
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
echo "Cleaning up..."
|
||||
crwl server stop
|
||||
|
||||
echo ""
|
||||
echo "✅ Test passed: Server started on custom port 8080"
|
||||
54
deploy/docker/tests/cli/basic/test_05_start_replicas.sh
Executable file
54
deploy/docker/tests/cli/basic/test_05_start_replicas.sh
Executable file
@@ -0,0 +1,54 @@
|
||||
#!/bin/bash
|
||||
# Test: Start server with multiple replicas
|
||||
# Expected: Server starts with 3 replicas in compose mode
|
||||
|
||||
set -e
|
||||
|
||||
echo "=== Test: Start Server with 3 Replicas ==="
|
||||
echo ""
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
|
||||
source "$PROJECT_ROOT/venv/bin/activate"
|
||||
|
||||
# Cleanup
|
||||
crwl server stop 2>/dev/null || true
|
||||
sleep 2
|
||||
|
||||
# Start with 3 replicas
|
||||
echo "Starting server with 3 replicas..."
|
||||
crwl server start --replicas 3
|
||||
|
||||
sleep 10
|
||||
|
||||
# Check status shows 3 replicas
|
||||
echo "Checking status..."
|
||||
STATUS_OUTPUT=$(crwl server status)
|
||||
echo "$STATUS_OUTPUT"
|
||||
|
||||
if ! echo "$STATUS_OUTPUT" | grep -q "3"; then
|
||||
echo "❌ Status does not show 3 replicas"
|
||||
crwl server stop
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check health endpoint
|
||||
echo "Checking health endpoint..."
|
||||
HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
|
||||
if [[ "$HEALTH" != "ok" ]]; then
|
||||
echo "❌ Health check failed"
|
||||
crwl server stop
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check container discovery (should show 3 containers eventually)
|
||||
echo "Checking container discovery..."
|
||||
sleep 5 # Wait for heartbeats
|
||||
CONTAINERS=$(curl -s http://localhost:11235/monitor/containers | jq -r '.count' 2>/dev/null || echo "0")
|
||||
echo "Container count: $CONTAINERS"
|
||||
|
||||
# Cleanup
|
||||
echo "Cleaning up..."
|
||||
crwl server stop
|
||||
|
||||
echo ""
|
||||
echo "✅ Test passed: Server started with 3 replicas"
|
||||
47
deploy/docker/tests/cli/basic/test_06_logs.sh
Executable file
47
deploy/docker/tests/cli/basic/test_06_logs.sh
Executable file
@@ -0,0 +1,47 @@
|
||||
#!/bin/bash
|
||||
# Test: View server logs
|
||||
# Expected: Logs are displayed without errors
|
||||
|
||||
set -e
|
||||
|
||||
echo "=== Test: Server Logs Command ==="
|
||||
echo ""
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
|
||||
source "$PROJECT_ROOT/venv/bin/activate"
|
||||
|
||||
# Start server
|
||||
echo "Starting server..."
|
||||
crwl server start >/dev/null 2>&1
|
||||
sleep 5
|
||||
|
||||
# Make a request to generate some logs
|
||||
echo "Making request to generate logs..."
|
||||
curl -s http://localhost:11235/health > /dev/null
|
||||
|
||||
# Check logs (tail)
|
||||
echo "Fetching logs (last 50 lines)..."
|
||||
LOGS=$(crwl server logs --tail 50 2>&1 || echo "ERROR")
|
||||
if [[ "$LOGS" == "ERROR" ]]; then
|
||||
echo "❌ Failed to retrieve logs"
|
||||
crwl server stop
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Log sample (first 10 lines):"
|
||||
echo "$LOGS" | head -n 10
|
||||
echo ""
|
||||
|
||||
# Verify logs contain something (not empty)
|
||||
if [[ -z "$LOGS" ]]; then
|
||||
echo "❌ Logs are empty"
|
||||
crwl server stop
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
echo "Cleaning up..."
|
||||
crwl server stop >/dev/null 2>&1
|
||||
|
||||
echo ""
|
||||
echo "✅ Test passed: Logs retrieved successfully"
|
||||
55
deploy/docker/tests/cli/basic/test_07_restart.sh
Executable file
55
deploy/docker/tests/cli/basic/test_07_restart.sh
Executable file
@@ -0,0 +1,55 @@
|
||||
#!/bin/bash
|
||||
# Test: Restart server command
|
||||
# Expected: Server restarts with same configuration
|
||||
|
||||
set -e
|
||||
|
||||
echo "=== Test: Restart Server Command ==="
|
||||
echo ""
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
|
||||
source "$PROJECT_ROOT/venv/bin/activate"
|
||||
|
||||
# Start server with specific config
|
||||
echo "Starting server with 2 replicas..."
|
||||
crwl server start --replicas 2 >/dev/null 2>&1
|
||||
sleep 8
|
||||
|
||||
# Get initial container ID
|
||||
echo "Getting initial state..."
|
||||
INITIAL_STATUS=$(crwl server status)
|
||||
echo "$INITIAL_STATUS"
|
||||
|
||||
# Restart
|
||||
echo ""
|
||||
echo "Restarting server..."
|
||||
crwl server restart
|
||||
|
||||
sleep 8
|
||||
|
||||
# Check status after restart
|
||||
echo "Checking status after restart..."
|
||||
RESTART_STATUS=$(crwl server status)
|
||||
echo "$RESTART_STATUS"
|
||||
|
||||
# Verify still has 2 replicas
|
||||
if ! echo "$RESTART_STATUS" | grep -q "2"; then
|
||||
echo "❌ Replica count not preserved after restart"
|
||||
crwl server stop
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify health
|
||||
HEALTH=$(curl -s http://localhost:11235/health | jq -r '.status' 2>/dev/null || echo "error")
|
||||
if [[ "$HEALTH" != "ok" ]]; then
|
||||
echo "❌ Health check failed after restart"
|
||||
crwl server stop
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
echo "Cleaning up..."
|
||||
crwl server stop >/dev/null 2>&1
|
||||
|
||||
echo ""
|
||||
echo "✅ Test passed: Server restarted with preserved configuration"
|
||||
46
deploy/docker/tests/cli/basic/test_08_cleanup.sh
Executable file
46
deploy/docker/tests/cli/basic/test_08_cleanup.sh
Executable file
@@ -0,0 +1,46 @@
|
||||
#!/bin/bash
|
||||
# Test: Force cleanup command
|
||||
# Expected: All resources removed even if state is corrupted
|
||||
|
||||
set -e
|
||||
|
||||
echo "=== Test: Force Cleanup Command ==="
|
||||
echo ""
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
|
||||
source "$PROJECT_ROOT/venv/bin/activate"
|
||||
|
||||
# Start server
|
||||
echo "Starting server..."
|
||||
crwl server start >/dev/null 2>&1
|
||||
sleep 5
|
||||
|
||||
# Run cleanup (will prompt, so use force flag)
|
||||
echo "Running force cleanup..."
|
||||
crwl server cleanup --force
|
||||
|
||||
sleep 3
|
||||
|
||||
# Verify no containers running
|
||||
echo "Verifying cleanup..."
|
||||
CONTAINERS=$(docker ps --filter "name=crawl4ai" --format "{{.Names}}" || echo "")
|
||||
if [[ -n "$CONTAINERS" ]]; then
|
||||
echo "❌ Crawl4AI containers still running: $CONTAINERS"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify port is free
|
||||
if curl -s http://localhost:11235/health > /dev/null 2>&1; then
|
||||
echo "❌ Server still responding after cleanup"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify status shows not running
|
||||
STATUS=$(crwl server status | grep "No server" || echo "RUNNING")
|
||||
if [[ "$STATUS" == "RUNNING" ]]; then
|
||||
echo "❌ Status still shows server running after cleanup"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "✅ Test passed: Force cleanup removed all resources"
|
||||
Reference in New Issue
Block a user