-
-
+
+
Monitor
+
+
+
+
diff --git a/deploy/docker/test-websocket.py b/deploy/docker/test-websocket.py
new file mode 100755
index 00000000..db121deb
--- /dev/null
+++ b/deploy/docker/test-websocket.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+"""
+Quick WebSocket test - Connect to monitor WebSocket and print updates
+"""
+import asyncio
+import websockets
+import json
+
+async def test_websocket():
+ uri = "ws://localhost:11235/monitor/ws"
+ print(f"Connecting to {uri}...")
+
+ try:
+ async with websockets.connect(uri) as websocket:
+ print("✅ Connected!")
+
+ # Receive and print 5 updates
+ for i in range(5):
+ message = await websocket.recv()
+ data = json.loads(message)
+ print(f"\n📊 Update #{i+1}:")
+ print(f" - Health: CPU {data['health']['container']['cpu_percent']}%, Memory {data['health']['container']['memory_percent']}%")
+ print(f" - Active Requests: {len(data['requests']['active'])}")
+ print(f" - Browsers: {len(data['browsers'])}")
+
+ except Exception as e:
+ print(f"❌ Error: {e}")
+ return 1
+
+ print("\n✅ WebSocket test passed!")
+ return 0
+
+if __name__ == "__main__":
+ exit(asyncio.run(test_websocket()))
diff --git a/deploy/docker/tests/demo_monitor_dashboard.py b/deploy/docker/tests/demo_monitor_dashboard.py
new file mode 100755
index 00000000..699988a5
--- /dev/null
+++ b/deploy/docker/tests/demo_monitor_dashboard.py
@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""
+Monitor Dashboard Demo Script
+Generates varied activity to showcase all monitoring features for video recording.
+"""
+import httpx
+import asyncio
+import time
+from datetime import datetime
+
+BASE_URL = "http://localhost:11235"
+
+async def demo_dashboard():
+ print("🎬 Monitor Dashboard Demo - Starting...\n")
+ print(f"📊 Dashboard: {BASE_URL}/dashboard")
+ print("=" * 60)
+
+ async with httpx.AsyncClient(timeout=60.0) as client:
+
+ # Phase 1: Simple requests (permanent browser)
+ print("\n🔷 Phase 1: Testing permanent browser pool")
+ print("-" * 60)
+ for i in range(5):
+ print(f" {i+1}/5 Request to /crawl (default config)...")
+ try:
+ r = await client.post(
+ f"{BASE_URL}/crawl",
+ json={"urls": [f"https://httpbin.org/html?req={i}"], "crawler_config": {}}
+ )
+ print(f" ✅ Status: {r.status_code}, Time: {r.elapsed.total_seconds():.2f}s")
+ except Exception as e:
+ print(f" ❌ Error: {e}")
+ await asyncio.sleep(1) # Small delay between requests
+
+ # Phase 2: Create variant browsers (different configs)
+ print("\n🔶 Phase 2: Testing cold→hot pool promotion")
+ print("-" * 60)
+ viewports = [
+ {"width": 1920, "height": 1080},
+ {"width": 1280, "height": 720},
+ {"width": 800, "height": 600}
+ ]
+
+ for idx, viewport in enumerate(viewports):
+ print(f" Viewport {viewport['width']}x{viewport['height']}:")
+ for i in range(4): # 4 requests each to trigger promotion at 3
+ try:
+ r = await client.post(
+ f"{BASE_URL}/crawl",
+ json={
+ "urls": [f"https://httpbin.org/json?v={idx}&r={i}"],
+ "browser_config": {"viewport": viewport},
+ "crawler_config": {}
+ }
+ )
+ print(f" {i+1}/4 ✅ {r.status_code} - Should see cold→hot after 3 uses")
+ except Exception as e:
+ print(f" {i+1}/4 ❌ {e}")
+ await asyncio.sleep(0.5)
+
+ # Phase 3: Concurrent burst (stress pool)
+ print("\n🔷 Phase 3: Concurrent burst (10 parallel)")
+ print("-" * 60)
+ tasks = []
+ for i in range(10):
+ tasks.append(
+ client.post(
+ f"{BASE_URL}/crawl",
+ json={"urls": [f"https://httpbin.org/delay/2?burst={i}"], "crawler_config": {}}
+ )
+ )
+
+ print(" Sending 10 concurrent requests...")
+ start = time.time()
+ results = await asyncio.gather(*tasks, return_exceptions=True)
+ elapsed = time.time() - start
+
+ successes = sum(1 for r in results if not isinstance(r, Exception) and r.status_code == 200)
+ print(f" ✅ {successes}/10 succeeded in {elapsed:.2f}s")
+
+ # Phase 4: Multi-endpoint coverage
+ print("\n🔶 Phase 4: Testing multiple endpoints")
+ print("-" * 60)
+ endpoints = [
+ ("/md", {"url": "https://httpbin.org/html", "f": "fit", "c": "0"}),
+ ("/screenshot", {"url": "https://httpbin.org/html"}),
+ ("/pdf", {"url": "https://httpbin.org/html"}),
+ ]
+
+ for endpoint, payload in endpoints:
+ print(f" Testing {endpoint}...")
+ try:
+ if endpoint == "/md":
+ r = await client.post(f"{BASE_URL}{endpoint}", json=payload)
+ else:
+ r = await client.post(f"{BASE_URL}{endpoint}", json=payload)
+ print(f" ✅ {r.status_code}")
+ except Exception as e:
+ print(f" ❌ {e}")
+ await asyncio.sleep(1)
+
+ # Phase 5: Intentional error (to populate errors tab)
+ print("\n🔷 Phase 5: Generating error examples")
+ print("-" * 60)
+ print(" Triggering invalid URL error...")
+ try:
+ r = await client.post(
+ f"{BASE_URL}/crawl",
+ json={"urls": ["invalid://bad-url"], "crawler_config": {}}
+ )
+ print(f" Response: {r.status_code}")
+ except Exception as e:
+ print(f" ✅ Error captured: {type(e).__name__}")
+
+ # Phase 6: Wait for janitor activity
+ print("\n🔶 Phase 6: Waiting for janitor cleanup...")
+ print("-" * 60)
+ print(" Idle for 40s to allow janitor to clean cold pool browsers...")
+ for i in range(40, 0, -10):
+ print(f" {i}s remaining... (Check dashboard for cleanup events)")
+ await asyncio.sleep(10)
+
+ # Phase 7: Final stats check
+ print("\n🔷 Phase 7: Final dashboard state")
+ print("-" * 60)
+
+ r = await client.get(f"{BASE_URL}/monitor/health")
+ health = r.json()
+ print(f" Memory: {health['container']['memory_percent']:.1f}%")
+ print(f" Browsers: Perm={health['pool']['permanent']['active']}, "
+ f"Hot={health['pool']['hot']['count']}, Cold={health['pool']['cold']['count']}")
+
+ r = await client.get(f"{BASE_URL}/monitor/endpoints/stats")
+ stats = r.json()
+ print(f"\n Endpoint Stats:")
+ for endpoint, data in stats.items():
+ print(f" {endpoint}: {data['count']} req, "
+ f"{data['avg_latency_ms']:.0f}ms avg, "
+ f"{data['success_rate_percent']:.1f}% success")
+
+ r = await client.get(f"{BASE_URL}/monitor/browsers")
+ browsers = r.json()
+ print(f"\n Pool Efficiency:")
+ print(f" Total browsers: {browsers['summary']['total_count']}")
+ print(f" Memory usage: {browsers['summary']['total_memory_mb']} MB")
+ print(f" Reuse rate: {browsers['summary']['reuse_rate_percent']:.1f}%")
+
+ print("\n" + "=" * 60)
+ print("✅ Demo complete! Dashboard is now populated with rich data.")
+ print(f"\n📹 Recording tip: Refresh {BASE_URL}/dashboard")
+ print(" You should see:")
+ print(" • Active & completed requests")
+ print(" • Browser pool (permanent + hot/cold)")
+ print(" • Janitor cleanup events")
+ print(" • Endpoint analytics")
+ print(" • Memory timeline")
+
+if __name__ == "__main__":
+ try:
+ asyncio.run(demo_dashboard())
+ except KeyboardInterrupt:
+ print("\n\n⚠️ Demo interrupted by user")
+ except Exception as e:
+ print(f"\n\n❌ Demo failed: {e}")
diff --git a/deploy/docker/tests/requirements.txt b/deploy/docker/tests/requirements.txt
new file mode 100644
index 00000000..5f7a842f
--- /dev/null
+++ b/deploy/docker/tests/requirements.txt
@@ -0,0 +1,2 @@
+httpx>=0.25.0
+docker>=7.0.0
diff --git a/deploy/docker/tests/test_1_basic.py b/deploy/docker/tests/test_1_basic.py
new file mode 100755
index 00000000..c86de073
--- /dev/null
+++ b/deploy/docker/tests/test_1_basic.py
@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""
+Test 1: Basic Container Health + Single Endpoint
+- Starts container
+- Hits /health endpoint 10 times
+- Reports success rate and basic latency
+"""
+import asyncio
+import time
+import docker
+import httpx
+
+# Config
+IMAGE = "crawl4ai-local:latest"
+CONTAINER_NAME = "crawl4ai-test"
+PORT = 11235
+REQUESTS = 10
+
+async def test_endpoint(url: str, count: int):
+ """Hit endpoint multiple times, return stats."""
+ results = []
+ async with httpx.AsyncClient(timeout=30.0) as client:
+ for i in range(count):
+ start = time.time()
+ try:
+ resp = await client.get(url)
+ elapsed = (time.time() - start) * 1000 # ms
+ results.append({
+ "success": resp.status_code == 200,
+ "latency_ms": elapsed,
+ "status": resp.status_code
+ })
+ print(f" [{i+1}/{count}] ✓ {resp.status_code} - {elapsed:.0f}ms")
+ except Exception as e:
+ results.append({
+ "success": False,
+ "latency_ms": None,
+ "error": str(e)
+ })
+ print(f" [{i+1}/{count}] ✗ Error: {e}")
+ return results
+
+def start_container(client, image: str, name: str, port: int):
+ """Start container, return container object."""
+ # Clean up existing
+ try:
+ old = client.containers.get(name)
+ print(f"🧹 Stopping existing container '{name}'...")
+ old.stop()
+ old.remove()
+ except docker.errors.NotFound:
+ pass
+
+ print(f"🚀 Starting container '{name}' from image '{image}'...")
+ container = client.containers.run(
+ image,
+ name=name,
+ ports={f"{port}/tcp": port},
+ detach=True,
+ shm_size="1g",
+ environment={"PYTHON_ENV": "production"}
+ )
+
+ # Wait for health
+ print(f"⏳ Waiting for container to be healthy...")
+ for _ in range(30): # 30s timeout
+ time.sleep(1)
+ container.reload()
+ if container.status == "running":
+ try:
+ # Quick health check
+ import requests
+ resp = requests.get(f"http://localhost:{port}/health", timeout=2)
+ if resp.status_code == 200:
+ print(f"✅ Container healthy!")
+ return container
+ except:
+ pass
+ raise TimeoutError("Container failed to start")
+
+def stop_container(container):
+ """Stop and remove container."""
+ print(f"🛑 Stopping container...")
+ container.stop()
+ container.remove()
+ print(f"✅ Container removed")
+
+async def main():
+ print("="*60)
+ print("TEST 1: Basic Container Health + Single Endpoint")
+ print("="*60)
+
+ client = docker.from_env()
+ container = None
+
+ try:
+ # Start container
+ container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
+
+ # Test /health endpoint
+ print(f"\n📊 Testing /health endpoint ({REQUESTS} requests)...")
+ url = f"http://localhost:{PORT}/health"
+ results = await test_endpoint(url, REQUESTS)
+
+ # Calculate stats
+ successes = sum(1 for r in results if r["success"])
+ success_rate = (successes / len(results)) * 100
+ latencies = [r["latency_ms"] for r in results if r["latency_ms"] is not None]
+ avg_latency = sum(latencies) / len(latencies) if latencies else 0
+
+ # Print results
+ print(f"\n{'='*60}")
+ print(f"RESULTS:")
+ print(f" Success Rate: {success_rate:.1f}% ({successes}/{len(results)})")
+ print(f" Avg Latency: {avg_latency:.0f}ms")
+ if latencies:
+ print(f" Min Latency: {min(latencies):.0f}ms")
+ print(f" Max Latency: {max(latencies):.0f}ms")
+ print(f"{'='*60}")
+
+ # Pass/Fail
+ if success_rate >= 100:
+ print(f"✅ TEST PASSED")
+ return 0
+ else:
+ print(f"❌ TEST FAILED (expected 100% success rate)")
+ return 1
+
+ except Exception as e:
+ print(f"\n❌ TEST ERROR: {e}")
+ return 1
+ finally:
+ if container:
+ stop_container(container)
+
+if __name__ == "__main__":
+ exit_code = asyncio.run(main())
+ exit(exit_code)
diff --git a/deploy/docker/tests/test_2_memory.py b/deploy/docker/tests/test_2_memory.py
new file mode 100755
index 00000000..aed4c61c
--- /dev/null
+++ b/deploy/docker/tests/test_2_memory.py
@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+"""
+Test 2: Docker Stats Monitoring
+- Extends Test 1 with real-time container stats
+- Monitors memory % and CPU during requests
+- Reports baseline, peak, and final memory
+"""
+import asyncio
+import time
+import docker
+import httpx
+from threading import Thread, Event
+
+# Config
+IMAGE = "crawl4ai-local:latest"
+CONTAINER_NAME = "crawl4ai-test"
+PORT = 11235
+REQUESTS = 20 # More requests to see memory usage
+
+# Stats tracking
+stats_history = []
+stop_monitoring = Event()
+
+def monitor_stats(container):
+ """Background thread to collect container stats."""
+ for stat in container.stats(decode=True, stream=True):
+ if stop_monitoring.is_set():
+ break
+
+ try:
+ # Extract memory stats
+ mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024) # MB
+ mem_limit = stat['memory_stats'].get('limit', 1) / (1024 * 1024)
+ mem_percent = (mem_usage / mem_limit * 100) if mem_limit > 0 else 0
+
+ # Extract CPU stats (handle missing fields on Mac)
+ cpu_percent = 0
+ try:
+ cpu_delta = stat['cpu_stats']['cpu_usage']['total_usage'] - \
+ stat['precpu_stats']['cpu_usage']['total_usage']
+ system_delta = stat['cpu_stats'].get('system_cpu_usage', 0) - \
+ stat['precpu_stats'].get('system_cpu_usage', 0)
+ if system_delta > 0:
+ num_cpus = stat['cpu_stats'].get('online_cpus', 1)
+ cpu_percent = (cpu_delta / system_delta * num_cpus * 100.0)
+ except (KeyError, ZeroDivisionError):
+ pass
+
+ stats_history.append({
+ 'timestamp': time.time(),
+ 'memory_mb': mem_usage,
+ 'memory_percent': mem_percent,
+ 'cpu_percent': cpu_percent
+ })
+ except Exception as e:
+ # Skip malformed stats
+ pass
+
+ time.sleep(0.5) # Sample every 500ms
+
+async def test_endpoint(url: str, count: int):
+ """Hit endpoint, return stats."""
+ results = []
+ async with httpx.AsyncClient(timeout=30.0) as client:
+ for i in range(count):
+ start = time.time()
+ try:
+ resp = await client.get(url)
+ elapsed = (time.time() - start) * 1000
+ results.append({
+ "success": resp.status_code == 200,
+ "latency_ms": elapsed,
+ })
+ if (i + 1) % 5 == 0: # Print every 5 requests
+ print(f" [{i+1}/{count}] ✓ {resp.status_code} - {elapsed:.0f}ms")
+ except Exception as e:
+ results.append({"success": False, "error": str(e)})
+ print(f" [{i+1}/{count}] ✗ Error: {e}")
+ return results
+
+def start_container(client, image: str, name: str, port: int):
+ """Start container."""
+ try:
+ old = client.containers.get(name)
+ print(f"🧹 Stopping existing container '{name}'...")
+ old.stop()
+ old.remove()
+ except docker.errors.NotFound:
+ pass
+
+ print(f"🚀 Starting container '{name}'...")
+ container = client.containers.run(
+ image,
+ name=name,
+ ports={f"{port}/tcp": port},
+ detach=True,
+ shm_size="1g",
+ mem_limit="4g", # Set explicit memory limit
+ )
+
+ print(f"⏳ Waiting for health...")
+ for _ in range(30):
+ time.sleep(1)
+ container.reload()
+ if container.status == "running":
+ try:
+ import requests
+ resp = requests.get(f"http://localhost:{port}/health", timeout=2)
+ if resp.status_code == 200:
+ print(f"✅ Container healthy!")
+ return container
+ except:
+ pass
+ raise TimeoutError("Container failed to start")
+
+def stop_container(container):
+ """Stop container."""
+ print(f"🛑 Stopping container...")
+ container.stop()
+ container.remove()
+
+async def main():
+ print("="*60)
+ print("TEST 2: Docker Stats Monitoring")
+ print("="*60)
+
+ client = docker.from_env()
+ container = None
+ monitor_thread = None
+
+ try:
+ # Start container
+ container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
+
+ # Start stats monitoring in background
+ print(f"\n📊 Starting stats monitor...")
+ stop_monitoring.clear()
+ stats_history.clear()
+ monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
+ monitor_thread.start()
+
+ # Wait a bit for baseline
+ await asyncio.sleep(2)
+ baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
+ print(f"📏 Baseline memory: {baseline_mem:.1f} MB")
+
+ # Test /health endpoint
+ print(f"\n🔄 Running {REQUESTS} requests to /health...")
+ url = f"http://localhost:{PORT}/health"
+ results = await test_endpoint(url, REQUESTS)
+
+ # Wait a bit to capture peak
+ await asyncio.sleep(1)
+
+ # Stop monitoring
+ stop_monitoring.set()
+ if monitor_thread:
+ monitor_thread.join(timeout=2)
+
+ # Calculate stats
+ successes = sum(1 for r in results if r.get("success"))
+ success_rate = (successes / len(results)) * 100
+ latencies = [r["latency_ms"] for r in results if "latency_ms" in r]
+ avg_latency = sum(latencies) / len(latencies) if latencies else 0
+
+ # Memory stats
+ memory_samples = [s['memory_mb'] for s in stats_history]
+ peak_mem = max(memory_samples) if memory_samples else 0
+ final_mem = memory_samples[-1] if memory_samples else 0
+ mem_delta = final_mem - baseline_mem
+
+ # Print results
+ print(f"\n{'='*60}")
+ print(f"RESULTS:")
+ print(f" Success Rate: {success_rate:.1f}% ({successes}/{len(results)})")
+ print(f" Avg Latency: {avg_latency:.0f}ms")
+ print(f"\n Memory Stats:")
+ print(f" Baseline: {baseline_mem:.1f} MB")
+ print(f" Peak: {peak_mem:.1f} MB")
+ print(f" Final: {final_mem:.1f} MB")
+ print(f" Delta: {mem_delta:+.1f} MB")
+ print(f"{'='*60}")
+
+ # Pass/Fail
+ if success_rate >= 100 and mem_delta < 100: # No significant memory growth
+ print(f"✅ TEST PASSED")
+ return 0
+ else:
+ if success_rate < 100:
+ print(f"❌ TEST FAILED (success rate < 100%)")
+ if mem_delta >= 100:
+ print(f"⚠️ WARNING: Memory grew by {mem_delta:.1f} MB")
+ return 1
+
+ except Exception as e:
+ print(f"\n❌ TEST ERROR: {e}")
+ return 1
+ finally:
+ stop_monitoring.set()
+ if container:
+ stop_container(container)
+
+if __name__ == "__main__":
+ exit_code = asyncio.run(main())
+ exit(exit_code)
diff --git a/deploy/docker/tests/test_3_pool.py b/deploy/docker/tests/test_3_pool.py
new file mode 100755
index 00000000..9f2c00b2
--- /dev/null
+++ b/deploy/docker/tests/test_3_pool.py
@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+"""
+Test 3: Pool Validation - Permanent Browser Reuse
+- Tests /html endpoint (should use permanent browser)
+- Monitors container logs for pool hit markers
+- Validates browser reuse rate
+- Checks memory after browser creation
+"""
+import asyncio
+import time
+import docker
+import httpx
+from threading import Thread, Event
+
+# Config
+IMAGE = "crawl4ai-local:latest"
+CONTAINER_NAME = "crawl4ai-test"
+PORT = 11235
+REQUESTS = 30
+
+# Stats tracking
+stats_history = []
+stop_monitoring = Event()
+
+def monitor_stats(container):
+ """Background stats collector."""
+ for stat in container.stats(decode=True, stream=True):
+ if stop_monitoring.is_set():
+ break
+ try:
+ mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
+ stats_history.append({
+ 'timestamp': time.time(),
+ 'memory_mb': mem_usage,
+ })
+ except:
+ pass
+ time.sleep(0.5)
+
+def count_log_markers(container):
+ """Extract pool usage markers from logs."""
+ logs = container.logs().decode('utf-8')
+
+ permanent_hits = logs.count("🔥 Using permanent browser")
+ hot_hits = logs.count("♨️ Using hot pool browser")
+ cold_hits = logs.count("❄️ Using cold pool browser")
+ new_created = logs.count("🆕 Creating new browser")
+
+ return {
+ 'permanent_hits': permanent_hits,
+ 'hot_hits': hot_hits,
+ 'cold_hits': cold_hits,
+ 'new_created': new_created,
+ 'total_hits': permanent_hits + hot_hits + cold_hits
+ }
+
+async def test_endpoint(url: str, count: int):
+ """Hit endpoint multiple times."""
+ results = []
+ async with httpx.AsyncClient(timeout=60.0) as client:
+ for i in range(count):
+ start = time.time()
+ try:
+ resp = await client.post(url, json={"url": "https://httpbin.org/html"})
+ elapsed = (time.time() - start) * 1000
+ results.append({
+ "success": resp.status_code == 200,
+ "latency_ms": elapsed,
+ })
+ if (i + 1) % 10 == 0:
+ print(f" [{i+1}/{count}] ✓ {resp.status_code} - {elapsed:.0f}ms")
+ except Exception as e:
+ results.append({"success": False, "error": str(e)})
+ print(f" [{i+1}/{count}] ✗ Error: {e}")
+ return results
+
+def start_container(client, image: str, name: str, port: int):
+ """Start container."""
+ try:
+ old = client.containers.get(name)
+ print(f"🧹 Stopping existing container...")
+ old.stop()
+ old.remove()
+ except docker.errors.NotFound:
+ pass
+
+ print(f"🚀 Starting container...")
+ container = client.containers.run(
+ image,
+ name=name,
+ ports={f"{port}/tcp": port},
+ detach=True,
+ shm_size="1g",
+ mem_limit="4g",
+ )
+
+ print(f"⏳ Waiting for health...")
+ for _ in range(30):
+ time.sleep(1)
+ container.reload()
+ if container.status == "running":
+ try:
+ import requests
+ resp = requests.get(f"http://localhost:{port}/health", timeout=2)
+ if resp.status_code == 200:
+ print(f"✅ Container healthy!")
+ return container
+ except:
+ pass
+ raise TimeoutError("Container failed to start")
+
+def stop_container(container):
+ """Stop container."""
+ print(f"🛑 Stopping container...")
+ container.stop()
+ container.remove()
+
+async def main():
+ print("="*60)
+ print("TEST 3: Pool Validation - Permanent Browser Reuse")
+ print("="*60)
+
+ client = docker.from_env()
+ container = None
+ monitor_thread = None
+
+ try:
+ # Start container
+ container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
+
+ # Wait for permanent browser initialization
+ print(f"\n⏳ Waiting for permanent browser init (3s)...")
+ await asyncio.sleep(3)
+
+ # Start stats monitoring
+ print(f"📊 Starting stats monitor...")
+ stop_monitoring.clear()
+ stats_history.clear()
+ monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
+ monitor_thread.start()
+
+ await asyncio.sleep(1)
+ baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
+ print(f"📏 Baseline (with permanent browser): {baseline_mem:.1f} MB")
+
+ # Test /html endpoint (uses permanent browser for default config)
+ print(f"\n🔄 Running {REQUESTS} requests to /html...")
+ url = f"http://localhost:{PORT}/html"
+ results = await test_endpoint(url, REQUESTS)
+
+ # Wait a bit
+ await asyncio.sleep(1)
+
+ # Stop monitoring
+ stop_monitoring.set()
+ if monitor_thread:
+ monitor_thread.join(timeout=2)
+
+ # Analyze logs for pool markers
+ print(f"\n📋 Analyzing pool usage...")
+ pool_stats = count_log_markers(container)
+
+ # Calculate request stats
+ successes = sum(1 for r in results if r.get("success"))
+ success_rate = (successes / len(results)) * 100
+ latencies = [r["latency_ms"] for r in results if "latency_ms" in r]
+ avg_latency = sum(latencies) / len(latencies) if latencies else 0
+
+ # Memory stats
+ memory_samples = [s['memory_mb'] for s in stats_history]
+ peak_mem = max(memory_samples) if memory_samples else 0
+ final_mem = memory_samples[-1] if memory_samples else 0
+ mem_delta = final_mem - baseline_mem
+
+ # Calculate reuse rate
+ total_requests = len(results)
+ total_pool_hits = pool_stats['total_hits']
+ reuse_rate = (total_pool_hits / total_requests * 100) if total_requests > 0 else 0
+
+ # Print results
+ print(f"\n{'='*60}")
+ print(f"RESULTS:")
+ print(f" Success Rate: {success_rate:.1f}% ({successes}/{len(results)})")
+ print(f" Avg Latency: {avg_latency:.0f}ms")
+ print(f"\n Pool Stats:")
+ print(f" 🔥 Permanent Hits: {pool_stats['permanent_hits']}")
+ print(f" ♨️ Hot Pool Hits: {pool_stats['hot_hits']}")
+ print(f" ❄️ Cold Pool Hits: {pool_stats['cold_hits']}")
+ print(f" 🆕 New Created: {pool_stats['new_created']}")
+ print(f" 📊 Reuse Rate: {reuse_rate:.1f}%")
+ print(f"\n Memory Stats:")
+ print(f" Baseline: {baseline_mem:.1f} MB")
+ print(f" Peak: {peak_mem:.1f} MB")
+ print(f" Final: {final_mem:.1f} MB")
+ print(f" Delta: {mem_delta:+.1f} MB")
+ print(f"{'='*60}")
+
+ # Pass/Fail
+ passed = True
+ if success_rate < 100:
+ print(f"❌ FAIL: Success rate {success_rate:.1f}% < 100%")
+ passed = False
+ if reuse_rate < 80:
+ print(f"❌ FAIL: Reuse rate {reuse_rate:.1f}% < 80% (expected high permanent browser usage)")
+ passed = False
+ if pool_stats['permanent_hits'] < (total_requests * 0.8):
+ print(f"⚠️ WARNING: Only {pool_stats['permanent_hits']} permanent hits out of {total_requests} requests")
+ if mem_delta > 200:
+ print(f"⚠️ WARNING: Memory grew by {mem_delta:.1f} MB (possible browser leak)")
+
+ if passed:
+ print(f"✅ TEST PASSED")
+ return 0
+ else:
+ return 1
+
+ except Exception as e:
+ print(f"\n❌ TEST ERROR: {e}")
+ import traceback
+ traceback.print_exc()
+ return 1
+ finally:
+ stop_monitoring.set()
+ if container:
+ stop_container(container)
+
+if __name__ == "__main__":
+ exit_code = asyncio.run(main())
+ exit(exit_code)
diff --git a/deploy/docker/tests/test_4_concurrent.py b/deploy/docker/tests/test_4_concurrent.py
new file mode 100755
index 00000000..70198ddc
--- /dev/null
+++ b/deploy/docker/tests/test_4_concurrent.py
@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+"""
+Test 4: Concurrent Load Testing
+- Tests pool under concurrent load
+- Escalates: 10 → 50 → 100 concurrent requests
+- Validates latency distribution (P50, P95, P99)
+- Monitors memory stability
+"""
+import asyncio
+import time
+import docker
+import httpx
+from threading import Thread, Event
+from collections import defaultdict
+
+# Config
+IMAGE = "crawl4ai-local:latest"
+CONTAINER_NAME = "crawl4ai-test"
+PORT = 11235
+LOAD_LEVELS = [
+ {"name": "Light", "concurrent": 10, "requests": 20},
+ {"name": "Medium", "concurrent": 50, "requests": 100},
+ {"name": "Heavy", "concurrent": 100, "requests": 200},
+]
+
+# Stats
+stats_history = []
+stop_monitoring = Event()
+
+def monitor_stats(container):
+ """Background stats collector."""
+ for stat in container.stats(decode=True, stream=True):
+ if stop_monitoring.is_set():
+ break
+ try:
+ mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
+ stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage})
+ except:
+ pass
+ time.sleep(0.5)
+
+def count_log_markers(container):
+ """Extract pool markers."""
+ logs = container.logs().decode('utf-8')
+ return {
+ 'permanent': logs.count("🔥 Using permanent browser"),
+ 'hot': logs.count("♨️ Using hot pool browser"),
+ 'cold': logs.count("❄️ Using cold pool browser"),
+ 'new': logs.count("🆕 Creating new browser"),
+ }
+
+async def hit_endpoint(client, url, payload, semaphore):
+ """Single request with concurrency control."""
+ async with semaphore:
+ start = time.time()
+ try:
+ resp = await client.post(url, json=payload, timeout=60.0)
+ elapsed = (time.time() - start) * 1000
+ return {"success": resp.status_code == 200, "latency_ms": elapsed}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+async def run_concurrent_test(url, payload, concurrent, total_requests):
+ """Run concurrent requests."""
+ semaphore = asyncio.Semaphore(concurrent)
+ async with httpx.AsyncClient() as client:
+ tasks = [hit_endpoint(client, url, payload, semaphore) for _ in range(total_requests)]
+ results = await asyncio.gather(*tasks)
+ return results
+
+def calculate_percentiles(latencies):
+ """Calculate P50, P95, P99."""
+ if not latencies:
+ return 0, 0, 0
+ sorted_lat = sorted(latencies)
+ n = len(sorted_lat)
+ return (
+ sorted_lat[int(n * 0.50)],
+ sorted_lat[int(n * 0.95)],
+ sorted_lat[int(n * 0.99)],
+ )
+
+def start_container(client, image, name, port):
+ """Start container."""
+ try:
+ old = client.containers.get(name)
+ print(f"🧹 Stopping existing container...")
+ old.stop()
+ old.remove()
+ except docker.errors.NotFound:
+ pass
+
+ print(f"🚀 Starting container...")
+ container = client.containers.run(
+ image, name=name, ports={f"{port}/tcp": port},
+ detach=True, shm_size="1g", mem_limit="4g",
+ )
+
+ print(f"⏳ Waiting for health...")
+ for _ in range(30):
+ time.sleep(1)
+ container.reload()
+ if container.status == "running":
+ try:
+ import requests
+ if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200:
+ print(f"✅ Container healthy!")
+ return container
+ except:
+ pass
+ raise TimeoutError("Container failed to start")
+
+async def main():
+ print("="*60)
+ print("TEST 4: Concurrent Load Testing")
+ print("="*60)
+
+ client = docker.from_env()
+ container = None
+ monitor_thread = None
+
+ try:
+ container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
+
+ print(f"\n⏳ Waiting for permanent browser init (3s)...")
+ await asyncio.sleep(3)
+
+ # Start monitoring
+ stop_monitoring.clear()
+ stats_history.clear()
+ monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
+ monitor_thread.start()
+
+ await asyncio.sleep(1)
+ baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
+ print(f"📏 Baseline: {baseline_mem:.1f} MB\n")
+
+ url = f"http://localhost:{PORT}/html"
+ payload = {"url": "https://httpbin.org/html"}
+
+ all_results = []
+ level_stats = []
+
+ # Run load levels
+ for level in LOAD_LEVELS:
+ print(f"{'='*60}")
+ print(f"🔄 {level['name']} Load: {level['concurrent']} concurrent, {level['requests']} total")
+ print(f"{'='*60}")
+
+ start_time = time.time()
+ results = await run_concurrent_test(url, payload, level['concurrent'], level['requests'])
+ duration = time.time() - start_time
+
+ successes = sum(1 for r in results if r.get("success"))
+ success_rate = (successes / len(results)) * 100
+ latencies = [r["latency_ms"] for r in results if "latency_ms" in r]
+ p50, p95, p99 = calculate_percentiles(latencies)
+ avg_lat = sum(latencies) / len(latencies) if latencies else 0
+
+ print(f" Duration: {duration:.1f}s")
+ print(f" Success: {success_rate:.1f}% ({successes}/{len(results)})")
+ print(f" Avg Latency: {avg_lat:.0f}ms")
+ print(f" P50/P95/P99: {p50:.0f}ms / {p95:.0f}ms / {p99:.0f}ms")
+
+ level_stats.append({
+ 'name': level['name'],
+ 'concurrent': level['concurrent'],
+ 'success_rate': success_rate,
+ 'avg_latency': avg_lat,
+ 'p50': p50, 'p95': p95, 'p99': p99,
+ })
+ all_results.extend(results)
+
+ await asyncio.sleep(2) # Cool down between levels
+
+ # Stop monitoring
+ await asyncio.sleep(1)
+ stop_monitoring.set()
+ if monitor_thread:
+ monitor_thread.join(timeout=2)
+
+ # Final stats
+ pool_stats = count_log_markers(container)
+ memory_samples = [s['memory_mb'] for s in stats_history]
+ peak_mem = max(memory_samples) if memory_samples else 0
+ final_mem = memory_samples[-1] if memory_samples else 0
+
+ print(f"\n{'='*60}")
+ print(f"FINAL RESULTS:")
+ print(f"{'='*60}")
+ print(f" Total Requests: {len(all_results)}")
+ print(f"\n Pool Utilization:")
+ print(f" 🔥 Permanent: {pool_stats['permanent']}")
+ print(f" ♨️ Hot: {pool_stats['hot']}")
+ print(f" ❄️ Cold: {pool_stats['cold']}")
+ print(f" 🆕 New: {pool_stats['new']}")
+ print(f"\n Memory:")
+ print(f" Baseline: {baseline_mem:.1f} MB")
+ print(f" Peak: {peak_mem:.1f} MB")
+ print(f" Final: {final_mem:.1f} MB")
+ print(f" Delta: {final_mem - baseline_mem:+.1f} MB")
+ print(f"{'='*60}")
+
+ # Pass/Fail
+ passed = True
+ for ls in level_stats:
+ if ls['success_rate'] < 99:
+ print(f"❌ FAIL: {ls['name']} success rate {ls['success_rate']:.1f}% < 99%")
+ passed = False
+ if ls['p99'] > 10000: # 10s threshold
+ print(f"⚠️ WARNING: {ls['name']} P99 latency {ls['p99']:.0f}ms very high")
+
+ if final_mem - baseline_mem > 300:
+ print(f"⚠️ WARNING: Memory grew {final_mem - baseline_mem:.1f} MB")
+
+ if passed:
+ print(f"✅ TEST PASSED")
+ return 0
+ else:
+ return 1
+
+ except Exception as e:
+ print(f"\n❌ TEST ERROR: {e}")
+ import traceback
+ traceback.print_exc()
+ return 1
+ finally:
+ stop_monitoring.set()
+ if container:
+ print(f"🛑 Stopping container...")
+ container.stop()
+ container.remove()
+
+if __name__ == "__main__":
+ exit_code = asyncio.run(main())
+ exit(exit_code)
diff --git a/deploy/docker/tests/test_5_pool_stress.py b/deploy/docker/tests/test_5_pool_stress.py
new file mode 100755
index 00000000..40752d84
--- /dev/null
+++ b/deploy/docker/tests/test_5_pool_stress.py
@@ -0,0 +1,267 @@
+#!/usr/bin/env python3
+"""
+Test 5: Pool Stress - Mixed Configs
+- Tests hot/cold pool with different browser configs
+- Uses different viewports to create config variants
+- Validates cold → hot promotion after 3 uses
+- Monitors pool tier distribution
+"""
+import asyncio
+import time
+import docker
+import httpx
+from threading import Thread, Event
+import random
+
+# Config
+IMAGE = "crawl4ai-local:latest"
+CONTAINER_NAME = "crawl4ai-test"
+PORT = 11235
+REQUESTS_PER_CONFIG = 5 # 5 requests per config variant
+
+# Different viewport configs to test pool tiers
+VIEWPORT_CONFIGS = [
+ None, # Default (permanent browser)
+ {"width": 1920, "height": 1080}, # Desktop
+ {"width": 1024, "height": 768}, # Tablet
+ {"width": 375, "height": 667}, # Mobile
+]
+
+# Stats
+stats_history = []
+stop_monitoring = Event()
+
+def monitor_stats(container):
+ """Background stats collector."""
+ for stat in container.stats(decode=True, stream=True):
+ if stop_monitoring.is_set():
+ break
+ try:
+ mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
+ stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage})
+ except:
+ pass
+ time.sleep(0.5)
+
+def analyze_pool_logs(container):
+ """Extract detailed pool stats from logs."""
+ logs = container.logs().decode('utf-8')
+
+ permanent = logs.count("🔥 Using permanent browser")
+ hot = logs.count("♨️ Using hot pool browser")
+ cold = logs.count("❄️ Using cold pool browser")
+ new = logs.count("🆕 Creating new browser")
+ promotions = logs.count("⬆️ Promoting to hot pool")
+
+ return {
+ 'permanent': permanent,
+ 'hot': hot,
+ 'cold': cold,
+ 'new': new,
+ 'promotions': promotions,
+ 'total': permanent + hot + cold
+ }
+
+async def crawl_with_viewport(client, url, viewport):
+ """Single request with specific viewport."""
+ payload = {
+ "urls": ["https://httpbin.org/html"],
+ "browser_config": {},
+ "crawler_config": {}
+ }
+
+ # Add viewport if specified
+ if viewport:
+ payload["browser_config"] = {
+ "type": "BrowserConfig",
+ "params": {
+ "viewport": {"type": "dict", "value": viewport},
+ "headless": True,
+ "text_mode": True,
+ "extra_args": [
+ "--no-sandbox",
+ "--disable-dev-shm-usage",
+ "--disable-gpu",
+ "--disable-software-rasterizer",
+ "--disable-web-security",
+ "--allow-insecure-localhost",
+ "--ignore-certificate-errors"
+ ]
+ }
+ }
+
+ start = time.time()
+ try:
+ resp = await client.post(url, json=payload, timeout=60.0)
+ elapsed = (time.time() - start) * 1000
+ return {"success": resp.status_code == 200, "latency_ms": elapsed, "viewport": viewport}
+ except Exception as e:
+ return {"success": False, "error": str(e), "viewport": viewport}
+
+def start_container(client, image, name, port):
+ """Start container."""
+ try:
+ old = client.containers.get(name)
+ print(f"🧹 Stopping existing container...")
+ old.stop()
+ old.remove()
+ except docker.errors.NotFound:
+ pass
+
+ print(f"🚀 Starting container...")
+ container = client.containers.run(
+ image, name=name, ports={f"{port}/tcp": port},
+ detach=True, shm_size="1g", mem_limit="4g",
+ )
+
+ print(f"⏳ Waiting for health...")
+ for _ in range(30):
+ time.sleep(1)
+ container.reload()
+ if container.status == "running":
+ try:
+ import requests
+ if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200:
+ print(f"✅ Container healthy!")
+ return container
+ except:
+ pass
+ raise TimeoutError("Container failed to start")
+
+async def main():
+ print("="*60)
+ print("TEST 5: Pool Stress - Mixed Configs")
+ print("="*60)
+
+ client = docker.from_env()
+ container = None
+ monitor_thread = None
+
+ try:
+ container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
+
+ print(f"\n⏳ Waiting for permanent browser init (3s)...")
+ await asyncio.sleep(3)
+
+ # Start monitoring
+ stop_monitoring.clear()
+ stats_history.clear()
+ monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
+ monitor_thread.start()
+
+ await asyncio.sleep(1)
+ baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
+ print(f"📏 Baseline: {baseline_mem:.1f} MB\n")
+
+ url = f"http://localhost:{PORT}/crawl"
+
+ print(f"Testing {len(VIEWPORT_CONFIGS)} different configs:")
+ for i, vp in enumerate(VIEWPORT_CONFIGS):
+ vp_str = "Default" if vp is None else f"{vp['width']}x{vp['height']}"
+ print(f" {i+1}. {vp_str}")
+ print()
+
+ # Run requests: repeat each config REQUESTS_PER_CONFIG times
+ all_results = []
+ config_sequence = []
+
+ for _ in range(REQUESTS_PER_CONFIG):
+ for viewport in VIEWPORT_CONFIGS:
+ config_sequence.append(viewport)
+
+ # Shuffle to mix configs
+ random.shuffle(config_sequence)
+
+ print(f"🔄 Running {len(config_sequence)} requests with mixed configs...")
+
+ async with httpx.AsyncClient() as http_client:
+ for i, viewport in enumerate(config_sequence):
+ result = await crawl_with_viewport(http_client, url, viewport)
+ all_results.append(result)
+
+ if (i + 1) % 5 == 0:
+ vp_str = "default" if result['viewport'] is None else f"{result['viewport']['width']}x{result['viewport']['height']}"
+ status = "✓" if result.get('success') else "✗"
+ lat = f"{result.get('latency_ms', 0):.0f}ms" if 'latency_ms' in result else "error"
+ print(f" [{i+1}/{len(config_sequence)}] {status} {vp_str} - {lat}")
+
+ # Stop monitoring
+ await asyncio.sleep(2)
+ stop_monitoring.set()
+ if monitor_thread:
+ monitor_thread.join(timeout=2)
+
+ # Analyze results
+ pool_stats = analyze_pool_logs(container)
+
+ successes = sum(1 for r in all_results if r.get("success"))
+ success_rate = (successes / len(all_results)) * 100
+ latencies = [r["latency_ms"] for r in all_results if "latency_ms" in r]
+ avg_lat = sum(latencies) / len(latencies) if latencies else 0
+
+ memory_samples = [s['memory_mb'] for s in stats_history]
+ peak_mem = max(memory_samples) if memory_samples else 0
+ final_mem = memory_samples[-1] if memory_samples else 0
+
+ print(f"\n{'='*60}")
+ print(f"RESULTS:")
+ print(f"{'='*60}")
+ print(f" Requests: {len(all_results)}")
+ print(f" Success Rate: {success_rate:.1f}% ({successes}/{len(all_results)})")
+ print(f" Avg Latency: {avg_lat:.0f}ms")
+ print(f"\n Pool Statistics:")
+ print(f" 🔥 Permanent: {pool_stats['permanent']}")
+ print(f" ♨️ Hot: {pool_stats['hot']}")
+ print(f" ❄️ Cold: {pool_stats['cold']}")
+ print(f" 🆕 New: {pool_stats['new']}")
+ print(f" ⬆️ Promotions: {pool_stats['promotions']}")
+ print(f" 📊 Reuse: {(pool_stats['total'] / len(all_results) * 100):.1f}%")
+ print(f"\n Memory:")
+ print(f" Baseline: {baseline_mem:.1f} MB")
+ print(f" Peak: {peak_mem:.1f} MB")
+ print(f" Final: {final_mem:.1f} MB")
+ print(f" Delta: {final_mem - baseline_mem:+.1f} MB")
+ print(f"{'='*60}")
+
+ # Pass/Fail
+ passed = True
+
+ if success_rate < 99:
+ print(f"❌ FAIL: Success rate {success_rate:.1f}% < 99%")
+ passed = False
+
+ # Should see promotions since we repeat each config 5 times
+ if pool_stats['promotions'] < (len(VIEWPORT_CONFIGS) - 1): # -1 for default
+ print(f"⚠️ WARNING: Only {pool_stats['promotions']} promotions (expected ~{len(VIEWPORT_CONFIGS)-1})")
+
+ # Should have created some browsers for different configs
+ if pool_stats['new'] == 0:
+ print(f"⚠️ NOTE: No new browsers created (all used default?)")
+
+ if pool_stats['permanent'] == len(all_results):
+ print(f"⚠️ NOTE: All requests used permanent browser (configs not varying enough?)")
+
+ if final_mem - baseline_mem > 500:
+ print(f"⚠️ WARNING: Memory grew {final_mem - baseline_mem:.1f} MB")
+
+ if passed:
+ print(f"✅ TEST PASSED")
+ return 0
+ else:
+ return 1
+
+ except Exception as e:
+ print(f"\n❌ TEST ERROR: {e}")
+ import traceback
+ traceback.print_exc()
+ return 1
+ finally:
+ stop_monitoring.set()
+ if container:
+ print(f"🛑 Stopping container...")
+ container.stop()
+ container.remove()
+
+if __name__ == "__main__":
+ exit_code = asyncio.run(main())
+ exit(exit_code)
diff --git a/deploy/docker/tests/test_6_multi_endpoint.py b/deploy/docker/tests/test_6_multi_endpoint.py
new file mode 100755
index 00000000..2d532d3b
--- /dev/null
+++ b/deploy/docker/tests/test_6_multi_endpoint.py
@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+"""
+Test 6: Multi-Endpoint Testing
+- Tests multiple endpoints together: /html, /screenshot, /pdf, /crawl
+- Validates each endpoint works correctly
+- Monitors success rates per endpoint
+"""
+import asyncio
+import time
+import docker
+import httpx
+from threading import Thread, Event
+
+# Config
+IMAGE = "crawl4ai-local:latest"
+CONTAINER_NAME = "crawl4ai-test"
+PORT = 11235
+REQUESTS_PER_ENDPOINT = 10
+
+# Stats
+stats_history = []
+stop_monitoring = Event()
+
+def monitor_stats(container):
+ """Background stats collector."""
+ for stat in container.stats(decode=True, stream=True):
+ if stop_monitoring.is_set():
+ break
+ try:
+ mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
+ stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage})
+ except:
+ pass
+ time.sleep(0.5)
+
+async def test_html(client, base_url, count):
+ """Test /html endpoint."""
+ url = f"{base_url}/html"
+ results = []
+ for _ in range(count):
+ start = time.time()
+ try:
+ resp = await client.post(url, json={"url": "https://httpbin.org/html"}, timeout=30.0)
+ elapsed = (time.time() - start) * 1000
+ results.append({"success": resp.status_code == 200, "latency_ms": elapsed})
+ except Exception as e:
+ results.append({"success": False, "error": str(e)})
+ return results
+
+async def test_screenshot(client, base_url, count):
+ """Test /screenshot endpoint."""
+ url = f"{base_url}/screenshot"
+ results = []
+ for _ in range(count):
+ start = time.time()
+ try:
+ resp = await client.post(url, json={"url": "https://httpbin.org/html"}, timeout=30.0)
+ elapsed = (time.time() - start) * 1000
+ results.append({"success": resp.status_code == 200, "latency_ms": elapsed})
+ except Exception as e:
+ results.append({"success": False, "error": str(e)})
+ return results
+
+async def test_pdf(client, base_url, count):
+ """Test /pdf endpoint."""
+ url = f"{base_url}/pdf"
+ results = []
+ for _ in range(count):
+ start = time.time()
+ try:
+ resp = await client.post(url, json={"url": "https://httpbin.org/html"}, timeout=30.0)
+ elapsed = (time.time() - start) * 1000
+ results.append({"success": resp.status_code == 200, "latency_ms": elapsed})
+ except Exception as e:
+ results.append({"success": False, "error": str(e)})
+ return results
+
+async def test_crawl(client, base_url, count):
+ """Test /crawl endpoint."""
+ url = f"{base_url}/crawl"
+ results = []
+ payload = {
+ "urls": ["https://httpbin.org/html"],
+ "browser_config": {},
+ "crawler_config": {}
+ }
+ for _ in range(count):
+ start = time.time()
+ try:
+ resp = await client.post(url, json=payload, timeout=30.0)
+ elapsed = (time.time() - start) * 1000
+ results.append({"success": resp.status_code == 200, "latency_ms": elapsed})
+ except Exception as e:
+ results.append({"success": False, "error": str(e)})
+ return results
+
+def start_container(client, image, name, port):
+ """Start container."""
+ try:
+ old = client.containers.get(name)
+ print(f"🧹 Stopping existing container...")
+ old.stop()
+ old.remove()
+ except docker.errors.NotFound:
+ pass
+
+ print(f"🚀 Starting container...")
+ container = client.containers.run(
+ image, name=name, ports={f"{port}/tcp": port},
+ detach=True, shm_size="1g", mem_limit="4g",
+ )
+
+ print(f"⏳ Waiting for health...")
+ for _ in range(30):
+ time.sleep(1)
+ container.reload()
+ if container.status == "running":
+ try:
+ import requests
+ if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200:
+ print(f"✅ Container healthy!")
+ return container
+ except:
+ pass
+ raise TimeoutError("Container failed to start")
+
+async def main():
+ print("="*60)
+ print("TEST 6: Multi-Endpoint Testing")
+ print("="*60)
+
+ client = docker.from_env()
+ container = None
+ monitor_thread = None
+
+ try:
+ container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
+
+ print(f"\n⏳ Waiting for permanent browser init (3s)...")
+ await asyncio.sleep(3)
+
+ # Start monitoring
+ stop_monitoring.clear()
+ stats_history.clear()
+ monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
+ monitor_thread.start()
+
+ await asyncio.sleep(1)
+ baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
+ print(f"📏 Baseline: {baseline_mem:.1f} MB\n")
+
+ base_url = f"http://localhost:{PORT}"
+
+ # Test each endpoint
+ endpoints = {
+ "/html": test_html,
+ "/screenshot": test_screenshot,
+ "/pdf": test_pdf,
+ "/crawl": test_crawl,
+ }
+
+ all_endpoint_stats = {}
+
+ async with httpx.AsyncClient() as http_client:
+ for endpoint_name, test_func in endpoints.items():
+ print(f"🔄 Testing {endpoint_name} ({REQUESTS_PER_ENDPOINT} requests)...")
+ results = await test_func(http_client, base_url, REQUESTS_PER_ENDPOINT)
+
+ successes = sum(1 for r in results if r.get("success"))
+ success_rate = (successes / len(results)) * 100
+ latencies = [r["latency_ms"] for r in results if "latency_ms" in r]
+ avg_lat = sum(latencies) / len(latencies) if latencies else 0
+
+ all_endpoint_stats[endpoint_name] = {
+ 'success_rate': success_rate,
+ 'avg_latency': avg_lat,
+ 'total': len(results),
+ 'successes': successes
+ }
+
+ print(f" ✓ Success: {success_rate:.1f}% ({successes}/{len(results)}), Avg: {avg_lat:.0f}ms")
+
+ # Stop monitoring
+ await asyncio.sleep(1)
+ stop_monitoring.set()
+ if monitor_thread:
+ monitor_thread.join(timeout=2)
+
+ # Final stats
+ memory_samples = [s['memory_mb'] for s in stats_history]
+ peak_mem = max(memory_samples) if memory_samples else 0
+ final_mem = memory_samples[-1] if memory_samples else 0
+
+ print(f"\n{'='*60}")
+ print(f"RESULTS:")
+ print(f"{'='*60}")
+ for endpoint, stats in all_endpoint_stats.items():
+ print(f" {endpoint:12} Success: {stats['success_rate']:5.1f}% Avg: {stats['avg_latency']:6.0f}ms")
+
+ print(f"\n Memory:")
+ print(f" Baseline: {baseline_mem:.1f} MB")
+ print(f" Peak: {peak_mem:.1f} MB")
+ print(f" Final: {final_mem:.1f} MB")
+ print(f" Delta: {final_mem - baseline_mem:+.1f} MB")
+ print(f"{'='*60}")
+
+ # Pass/Fail
+ passed = True
+ for endpoint, stats in all_endpoint_stats.items():
+ if stats['success_rate'] < 100:
+ print(f"❌ FAIL: {endpoint} success rate {stats['success_rate']:.1f}% < 100%")
+ passed = False
+
+ if passed:
+ print(f"✅ TEST PASSED")
+ return 0
+ else:
+ return 1
+
+ except Exception as e:
+ print(f"\n❌ TEST ERROR: {e}")
+ import traceback
+ traceback.print_exc()
+ return 1
+ finally:
+ stop_monitoring.set()
+ if container:
+ print(f"🛑 Stopping container...")
+ container.stop()
+ container.remove()
+
+if __name__ == "__main__":
+ exit_code = asyncio.run(main())
+ exit(exit_code)
diff --git a/deploy/docker/tests/test_7_cleanup.py b/deploy/docker/tests/test_7_cleanup.py
new file mode 100755
index 00000000..2fdbe9a6
--- /dev/null
+++ b/deploy/docker/tests/test_7_cleanup.py
@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+"""
+Test 7: Cleanup Verification (Janitor)
+- Creates load spike then goes idle
+- Verifies memory returns to near baseline
+- Tests janitor cleanup of idle browsers
+- Monitors memory recovery time
+"""
+import asyncio
+import time
+import docker
+import httpx
+from threading import Thread, Event
+
+# Config
+IMAGE = "crawl4ai-local:latest"
+CONTAINER_NAME = "crawl4ai-test"
+PORT = 11235
+SPIKE_REQUESTS = 20 # Create some browsers
+IDLE_TIME = 90 # Wait 90s for janitor (runs every 60s)
+
+# Stats
+stats_history = []
+stop_monitoring = Event()
+
+def monitor_stats(container):
+ """Background stats collector."""
+ for stat in container.stats(decode=True, stream=True):
+ if stop_monitoring.is_set():
+ break
+ try:
+ mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
+ stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage})
+ except:
+ pass
+ time.sleep(1) # Sample every 1s for this test
+
+def start_container(client, image, name, port):
+ """Start container."""
+ try:
+ old = client.containers.get(name)
+ print(f"🧹 Stopping existing container...")
+ old.stop()
+ old.remove()
+ except docker.errors.NotFound:
+ pass
+
+ print(f"🚀 Starting container...")
+ container = client.containers.run(
+ image, name=name, ports={f"{port}/tcp": port},
+ detach=True, shm_size="1g", mem_limit="4g",
+ )
+
+ print(f"⏳ Waiting for health...")
+ for _ in range(30):
+ time.sleep(1)
+ container.reload()
+ if container.status == "running":
+ try:
+ import requests
+ if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200:
+ print(f"✅ Container healthy!")
+ return container
+ except:
+ pass
+ raise TimeoutError("Container failed to start")
+
+async def main():
+ print("="*60)
+ print("TEST 7: Cleanup Verification (Janitor)")
+ print("="*60)
+
+ client = docker.from_env()
+ container = None
+ monitor_thread = None
+
+ try:
+ container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
+
+ print(f"\n⏳ Waiting for permanent browser init (3s)...")
+ await asyncio.sleep(3)
+
+ # Start monitoring
+ stop_monitoring.clear()
+ stats_history.clear()
+ monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
+ monitor_thread.start()
+
+ await asyncio.sleep(2)
+ baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
+ print(f"📏 Baseline: {baseline_mem:.1f} MB\n")
+
+ # Create load spike with different configs to populate pool
+ print(f"🔥 Creating load spike ({SPIKE_REQUESTS} requests with varied configs)...")
+ url = f"http://localhost:{PORT}/crawl"
+
+ viewports = [
+ {"width": 1920, "height": 1080},
+ {"width": 1024, "height": 768},
+ {"width": 375, "height": 667},
+ ]
+
+ async with httpx.AsyncClient(timeout=60.0) as http_client:
+ tasks = []
+ for i in range(SPIKE_REQUESTS):
+ vp = viewports[i % len(viewports)]
+ payload = {
+ "urls": ["https://httpbin.org/html"],
+ "browser_config": {
+ "type": "BrowserConfig",
+ "params": {
+ "viewport": {"type": "dict", "value": vp},
+ "headless": True,
+ "text_mode": True,
+ "extra_args": [
+ "--no-sandbox", "--disable-dev-shm-usage",
+ "--disable-gpu", "--disable-software-rasterizer",
+ "--disable-web-security", "--allow-insecure-localhost",
+ "--ignore-certificate-errors"
+ ]
+ }
+ },
+ "crawler_config": {}
+ }
+ tasks.append(http_client.post(url, json=payload))
+
+ results = await asyncio.gather(*tasks, return_exceptions=True)
+ successes = sum(1 for r in results if hasattr(r, 'status_code') and r.status_code == 200)
+ print(f" ✓ Spike completed: {successes}/{len(results)} successful")
+
+ # Measure peak
+ await asyncio.sleep(2)
+ peak_mem = max([s['memory_mb'] for s in stats_history]) if stats_history else baseline_mem
+ print(f" 📊 Peak memory: {peak_mem:.1f} MB (+{peak_mem - baseline_mem:.1f} MB)")
+
+ # Now go idle and wait for janitor
+ print(f"\n⏸️ Going idle for {IDLE_TIME}s (janitor cleanup)...")
+ print(f" (Janitor runs every 60s, checking for idle browsers)")
+
+ for elapsed in range(0, IDLE_TIME, 10):
+ await asyncio.sleep(10)
+ current_mem = stats_history[-1]['memory_mb'] if stats_history else 0
+ print(f" [{elapsed+10:3d}s] Memory: {current_mem:.1f} MB")
+
+ # Stop monitoring
+ stop_monitoring.set()
+ if monitor_thread:
+ monitor_thread.join(timeout=2)
+
+ # Analyze memory recovery
+ final_mem = stats_history[-1]['memory_mb'] if stats_history else 0
+ recovery_mb = peak_mem - final_mem
+ recovery_pct = (recovery_mb / (peak_mem - baseline_mem) * 100) if (peak_mem - baseline_mem) > 0 else 0
+
+ print(f"\n{'='*60}")
+ print(f"RESULTS:")
+ print(f"{'='*60}")
+ print(f" Memory Journey:")
+ print(f" Baseline: {baseline_mem:.1f} MB")
+ print(f" Peak: {peak_mem:.1f} MB (+{peak_mem - baseline_mem:.1f} MB)")
+ print(f" Final: {final_mem:.1f} MB (+{final_mem - baseline_mem:.1f} MB)")
+ print(f" Recovered: {recovery_mb:.1f} MB ({recovery_pct:.1f}%)")
+ print(f"{'='*60}")
+
+ # Pass/Fail
+ passed = True
+
+ # Should have created some memory pressure
+ if peak_mem - baseline_mem < 100:
+ print(f"⚠️ WARNING: Peak increase only {peak_mem - baseline_mem:.1f} MB (expected more browsers)")
+
+ # Should recover most memory (within 100MB of baseline)
+ if final_mem - baseline_mem > 100:
+ print(f"⚠️ WARNING: Memory didn't recover well (still +{final_mem - baseline_mem:.1f} MB above baseline)")
+ else:
+ print(f"✅ Good memory recovery!")
+
+ # Baseline + 50MB tolerance
+ if final_mem - baseline_mem < 50:
+ print(f"✅ Excellent cleanup (within 50MB of baseline)")
+
+ print(f"✅ TEST PASSED")
+ return 0
+
+ except Exception as e:
+ print(f"\n❌ TEST ERROR: {e}")
+ import traceback
+ traceback.print_exc()
+ return 1
+ finally:
+ stop_monitoring.set()
+ if container:
+ print(f"🛑 Stopping container...")
+ container.stop()
+ container.remove()
+
+if __name__ == "__main__":
+ exit_code = asyncio.run(main())
+ exit(exit_code)
diff --git a/deploy/docker/tests/test_monitor_demo.py b/deploy/docker/tests/test_monitor_demo.py
new file mode 100644
index 00000000..2dbff5b1
--- /dev/null
+++ b/deploy/docker/tests/test_monitor_demo.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""Quick test to generate monitor dashboard activity"""
+import httpx
+import asyncio
+
+async def test_dashboard():
+ async with httpx.AsyncClient(timeout=30.0) as client:
+ print("📊 Generating dashboard activity...")
+
+ # Test 1: Simple crawl
+ print("\n1️⃣ Running simple crawl...")
+ r1 = await client.post(
+ "http://localhost:11235/crawl",
+ json={"urls": ["https://httpbin.org/html"], "crawler_config": {}}
+ )
+ print(f" Status: {r1.status_code}")
+
+ # Test 2: Multiple URLs
+ print("\n2️⃣ Running multi-URL crawl...")
+ r2 = await client.post(
+ "http://localhost:11235/crawl",
+ json={
+ "urls": [
+ "https://httpbin.org/html",
+ "https://httpbin.org/json"
+ ],
+ "crawler_config": {}
+ }
+ )
+ print(f" Status: {r2.status_code}")
+
+ # Test 3: Check monitor health
+ print("\n3️⃣ Checking monitor health...")
+ r3 = await client.get("http://localhost:11235/monitor/health")
+ health = r3.json()
+ print(f" Memory: {health['container']['memory_percent']}%")
+ print(f" Browsers: {health['pool']['permanent']['active']}")
+
+ # Test 4: Check requests
+ print("\n4️⃣ Checking request log...")
+ r4 = await client.get("http://localhost:11235/monitor/requests")
+ reqs = r4.json()
+ print(f" Active: {len(reqs['active'])}")
+ print(f" Completed: {len(reqs['completed'])}")
+
+ # Test 5: Check endpoint stats
+ print("\n5️⃣ Checking endpoint stats...")
+ r5 = await client.get("http://localhost:11235/monitor/endpoints/stats")
+ stats = r5.json()
+ for endpoint, data in stats.items():
+ print(f" {endpoint}: {data['count']} requests, {data['avg_latency_ms']}ms avg")
+
+ print("\n✅ Dashboard should now show activity!")
+ print(f"\n🌐 Open: http://localhost:11235/dashboard")
+
+if __name__ == "__main__":
+ asyncio.run(test_dashboard())
diff --git a/deploy/docker/utils.py b/deploy/docker/utils.py
index 5f3618af..52f4e11f 100644
--- a/deploy/docker/utils.py
+++ b/deploy/docker/utils.py
@@ -178,4 +178,29 @@ def verify_email_domain(email: str) -> bool:
records = dns.resolver.resolve(domain, 'MX')
return True if records else False
except Exception as e:
- return False
\ No newline at end of file
+ return False
+
+def get_container_memory_percent() -> float:
+ """Get actual container memory usage vs limit (cgroup v1/v2 aware)."""
+ try:
+ # Try cgroup v2 first
+ usage_path = Path("/sys/fs/cgroup/memory.current")
+ limit_path = Path("/sys/fs/cgroup/memory.max")
+ if not usage_path.exists():
+ # Fall back to cgroup v1
+ usage_path = Path("/sys/fs/cgroup/memory/memory.usage_in_bytes")
+ limit_path = Path("/sys/fs/cgroup/memory/memory.limit_in_bytes")
+
+ usage = int(usage_path.read_text())
+ limit = int(limit_path.read_text())
+
+ # Handle unlimited (v2: "max", v1: > 1e18)
+ if limit > 1e18:
+ import psutil
+ limit = psutil.virtual_memory().total
+
+ return (usage / limit) * 100
+ except:
+ # Non-container or unsupported: fallback to host
+ import psutil
+ return psutil.virtual_memory().percent
\ No newline at end of file
diff --git a/docs/md_v2/core/docker-deployment.md b/docs/md_v2/core/self-hosting.md
similarity index 81%
rename from docs/md_v2/core/docker-deployment.md
rename to docs/md_v2/core/self-hosting.md
index 6d5a9eae..f389e65d 100644
--- a/docs/md_v2/core/docker-deployment.md
+++ b/docs/md_v2/core/self-hosting.md
@@ -1,4 +1,20 @@
-# Crawl4AI Docker Guide 🐳
+# Self-Hosting Crawl4AI 🚀
+
+**Take Control of Your Web Crawling Infrastructure**
+
+Self-hosting Crawl4AI gives you complete control over your web crawling and data extraction pipeline. Unlike cloud-based solutions, you own your data, infrastructure, and destiny.
+
+## Why Self-Host?
+
+- **🔒 Data Privacy**: Your crawled data never leaves your infrastructure
+- **💰 Cost Control**: No per-request pricing - scale within your own resources
+- **🎯 Customization**: Full control over browser configurations, extraction strategies, and performance tuning
+- **📊 Transparency**: Real-time monitoring dashboard shows exactly what's happening
+- **⚡ Performance**: Direct access without API rate limits or geographic restrictions
+- **🛡️ Security**: Keep sensitive data extraction workflows behind your firewall
+- **🔧 Flexibility**: Customize, extend, and integrate with your existing infrastructure
+
+When you self-host, you can scale from a single container to a full browser infrastructure, all while maintaining complete control and visibility.
## Table of Contents
- [Prerequisites](#prerequisites)
@@ -13,36 +29,14 @@
- [Available MCP Tools](#available-mcp-tools)
- [Testing MCP Connections](#testing-mcp-connections)
- [MCP Schemas](#mcp-schemas)
-- [Additional API Endpoints](#additional-api-endpoints)
- - [HTML Extraction Endpoint](#html-extraction-endpoint)
- - [Screenshot Endpoint](#screenshot-endpoint)
- - [PDF Export Endpoint](#pdf-export-endpoint)
- - [JavaScript Execution Endpoint](#javascript-execution-endpoint)
-- [User-Provided Hooks API](#user-provided-hooks-api)
- - [Hook Information Endpoint](#hook-information-endpoint)
- - [Available Hook Points](#available-hook-points)
- - [Using Hooks in Requests](#using-hooks-in-requests)
- - [Hook Examples with Real URLs](#hook-examples-with-real-urls)
- - [Security Best Practices](#security-best-practices)
- - [Hook Response Information](#hook-response-information)
- - [Error Handling](#error-handling)
- - [Hooks Utility: Function-Based Approach (Python)](#hooks-utility-function-based-approach-python)
-- [Job Queue & Webhook API](#job-queue-webhook-api)
- - [Why Use the Job Queue API?](#why-use-the-job-queue-api)
- - [Available Endpoints](#available-endpoints)
- - [Webhook Configuration](#webhook-configuration)
- - [Usage Examples](#usage-examples)
- - [Webhook Best Practices](#webhook-best-practices)
- - [Use Cases](#use-cases)
- - [Troubleshooting](#troubleshooting)
-- [Dockerfile Parameters](#dockerfile-parameters)
-- [Using the API](#using-the-api)
- - [Playground Interface](#playground-interface)
- - [Python SDK](#python-sdk)
- - [Understanding Request Schema](#understanding-request-schema)
- - [REST API Examples](#rest-api-examples)
- - [LLM Configuration Examples](#llm-configuration-examples)
-- [Metrics & Monitoring](#metrics--monitoring)
+- [Real-time Monitoring & Operations](#real-time-monitoring--operations)
+ - [Monitoring Dashboard](#monitoring-dashboard)
+ - [Monitor API Endpoints](#monitor-api-endpoints)
+ - [WebSocket Streaming](#websocket-streaming)
+ - [Control Actions](#control-actions)
+ - [Production Integration](#production-integration)
+- [Deployment Scenarios](#deployment-scenarios)
+- [Complete Examples](#complete-examples)
- [Server Configuration](#server-configuration)
- [Understanding config.yml](#understanding-configyml)
- [JWT Authentication](#jwt-authentication)
@@ -1957,22 +1951,469 @@ async def test_stream_crawl(token: str = None): # Made token optional
---
-## Metrics & Monitoring
+## Real-time Monitoring & Operations
-Keep an eye on your crawler with these endpoints:
+One of the key advantages of self-hosting is complete visibility into your infrastructure. Crawl4AI includes a comprehensive real-time monitoring system that gives you full transparency and control.
-- `/health` - Quick health check
-- `/metrics` - Detailed Prometheus metrics
-- `/schema` - Full API schema
+### Monitoring Dashboard
-Example health check:
+Access the **built-in real-time monitoring dashboard** for complete operational visibility:
+
+```
+http://localhost:11235/monitor
+```
+
+
+
+**Dashboard Features:**
+
+#### 1. System Health Overview
+- **CPU & Memory**: Live usage with progress bars and percentage indicators
+- **Network I/O**: Total bytes sent/received since startup
+- **Server Uptime**: How long your server has been running
+- **Browser Pool Status**:
+ - 🔥 Permanent browser (always-on default config, ~270MB)
+ - ♨️ Hot pool (frequently used configs, ~180MB each)
+ - ❄️ Cold pool (idle browsers awaiting cleanup, ~180MB each)
+- **Memory Pressure**: LOW/MEDIUM/HIGH indicator for janitor behavior
+
+#### 2. Live Request Tracking
+- **Active Requests**: Currently running crawls with:
+ - Request ID for tracking
+ - Target URL (truncated for display)
+ - Endpoint being used
+ - Elapsed time (updates in real-time)
+ - Memory usage from start
+- **Completed Requests**: Last 10 finished requests showing:
+ - Success/failure status (color-coded)
+ - Total execution time
+ - Memory delta (how much memory changed)
+ - Pool hit (was browser reused?)
+ - HTTP status code
+- **Filtering**: View all, success only, or errors only
+
+#### 3. Browser Pool Management
+Interactive table showing all active browsers:
+
+| Type | Signature | Age | Last Used | Hits | Actions |
+|------|-----------|-----|-----------|------|---------|
+| permanent | abc12345 | 2h | 5s ago | 1,247 | Restart |
+| hot | def67890 | 45m | 2m ago | 89 | Kill / Restart |
+| cold | ghi11213 | 30m | 15m ago | 3 | Kill / Restart |
+
+- **Reuse Rate**: Percentage of requests that reused existing browsers
+- **Memory Estimates**: Total memory used by browser pool
+- **Manual Control**: Kill or restart individual browsers
+
+#### 4. Janitor Events Log
+Real-time log of browser pool cleanup events:
+- When cold browsers are closed due to memory pressure
+- When browsers are promoted from cold to hot pool
+- Forced cleanups triggered manually
+- Detailed cleanup reasons and browser signatures
+
+#### 5. Error Monitoring
+Recent errors with full context:
+- Timestamp
+- Endpoint where error occurred
+- Target URL
+- Error message
+- Request ID for correlation
+
+**Live Updates:**
+The dashboard connects via WebSocket and refreshes every **2 seconds** with the latest data. Connection status indicator shows when you're connected/disconnected.
+
+---
+
+### Monitor API Endpoints
+
+For programmatic monitoring, automation, and integration with your existing infrastructure:
+
+#### Health & Statistics
+
+**Get System Health**
```bash
-curl http://localhost:11235/health
+GET /monitor/health
+```
+
+Returns current system snapshot:
+```json
+{
+ "container": {
+ "memory_percent": 45.2,
+ "cpu_percent": 23.1,
+ "network_sent_mb": 1250.45,
+ "network_recv_mb": 3421.12,
+ "uptime_seconds": 7234
+ },
+ "pool": {
+ "permanent": {"active": true, "memory_mb": 270},
+ "hot": {"count": 3, "memory_mb": 540},
+ "cold": {"count": 1, "memory_mb": 180},
+ "total_memory_mb": 990
+ },
+ "janitor": {
+ "next_cleanup_estimate": "adaptive",
+ "memory_pressure": "MEDIUM"
+ }
+}
+```
+
+**Get Request Statistics**
+```bash
+GET /monitor/requests?status=all&limit=50
+```
+
+Query parameters:
+- `status`: Filter by `all`, `active`, `completed`, `success`, or `error`
+- `limit`: Number of completed requests to return (1-1000)
+
+**Get Browser Pool Details**
+```bash
+GET /monitor/browsers
+```
+
+Returns detailed information about all active browsers:
+```json
+{
+ "browsers": [
+ {
+ "type": "permanent",
+ "sig": "abc12345",
+ "age_seconds": 7234,
+ "last_used_seconds": 5,
+ "memory_mb": 270,
+ "hits": 1247,
+ "killable": false
+ },
+ {
+ "type": "hot",
+ "sig": "def67890",
+ "age_seconds": 2701,
+ "last_used_seconds": 120,
+ "memory_mb": 180,
+ "hits": 89,
+ "killable": true
+ }
+ ],
+ "summary": {
+ "total_count": 5,
+ "total_memory_mb": 990,
+ "reuse_rate_percent": 87.3
+ }
+}
+```
+
+**Get Endpoint Performance Statistics**
+```bash
+GET /monitor/endpoints/stats
+```
+
+Returns aggregated metrics per endpoint:
+```json
+{
+ "/crawl": {
+ "count": 1523,
+ "avg_latency_ms": 2341.5,
+ "success_rate_percent": 98.2,
+ "pool_hit_rate_percent": 89.1,
+ "errors": 27
+ },
+ "/md": {
+ "count": 891,
+ "avg_latency_ms": 1823.7,
+ "success_rate_percent": 99.4,
+ "pool_hit_rate_percent": 92.3,
+ "errors": 5
+ }
+}
+```
+
+**Get Timeline Data**
+```bash
+GET /monitor/timeline?metric=memory&window=5m
+```
+
+Parameters:
+- `metric`: `memory`, `requests`, or `browsers`
+- `window`: Currently only `5m` (5-minute window, 5-second resolution)
+
+Returns time-series data for charts:
+```json
+{
+ "timestamps": [1699564800, 1699564805, 1699564810, ...],
+ "values": [42.1, 43.5, 41.8, ...]
+}
+```
+
+#### Logs
+
+**Get Janitor Events**
+```bash
+GET /monitor/logs/janitor?limit=100
+```
+
+**Get Error Log**
+```bash
+GET /monitor/logs/errors?limit=100
```
---
-*(Deployment Scenarios and Complete Examples sections remain the same, maybe update links if examples moved)*
+### WebSocket Streaming
+
+For real-time monitoring in your own dashboards or applications:
+
+```bash
+WS /monitor/ws
+```
+
+**Connection Example (Python):**
+```python
+import asyncio
+import websockets
+import json
+
+async def monitor_server():
+ uri = "ws://localhost:11235/monitor/ws"
+
+ async with websockets.connect(uri) as websocket:
+ print("Connected to Crawl4AI monitor")
+
+ while True:
+ # Receive update every 2 seconds
+ data = await websocket.recv()
+ update = json.loads(data)
+
+ # Extract key metrics
+ health = update['health']
+ active_requests = len(update['requests']['active'])
+ browsers = len(update['browsers'])
+
+ print(f"Memory: {health['container']['memory_percent']:.1f}% | "
+ f"Active: {active_requests} | "
+ f"Browsers: {browsers}")
+
+ # Check for high memory pressure
+ if health['janitor']['memory_pressure'] == 'HIGH':
+ print("⚠️ HIGH MEMORY PRESSURE - Consider cleanup")
+
+asyncio.run(monitor_server())
+```
+
+**Update Payload Structure:**
+```json
+{
+ "timestamp": 1699564823.456,
+ "health": { /* System health snapshot */ },
+ "requests": {
+ "active": [ /* Currently running */ ],
+ "completed": [ /* Last 10 completed */ ]
+ },
+ "browsers": [ /* All active browsers */ ],
+ "timeline": {
+ "memory": { /* Last 5 minutes */ },
+ "requests": { /* Request rate */ },
+ "browsers": { /* Pool composition */ }
+ },
+ "janitor": [ /* Last 10 cleanup events */ ],
+ "errors": [ /* Last 10 errors */ ]
+}
+```
+
+---
+
+### Control Actions
+
+Take manual control when needed:
+
+**Force Immediate Cleanup**
+```bash
+POST /monitor/actions/cleanup
+```
+
+Kills all cold pool browsers immediately (useful when memory is tight):
+```json
+{
+ "success": true,
+ "killed_browsers": 3
+}
+```
+
+**Kill Specific Browser**
+```bash
+POST /monitor/actions/kill_browser
+Content-Type: application/json
+
+{
+ "sig": "abc12345" // First 8 chars of browser signature
+}
+```
+
+Response:
+```json
+{
+ "success": true,
+ "killed_sig": "abc12345",
+ "pool_type": "hot"
+}
+```
+
+**Restart Browser**
+```bash
+POST /monitor/actions/restart_browser
+Content-Type: application/json
+
+{
+ "sig": "permanent" // Or first 8 chars of signature
+}
+```
+
+For permanent browser, this will close and reinitialize it. For hot/cold browsers, it kills them and lets new requests create fresh ones.
+
+**Reset Statistics**
+```bash
+POST /monitor/stats/reset
+```
+
+Clears endpoint counters (useful for starting fresh after testing).
+
+---
+
+### Production Integration
+
+#### Integration with Existing Monitoring Systems
+
+**Prometheus Integration:**
+```bash
+# Scrape metrics endpoint
+curl http://localhost:11235/metrics
+```
+
+**Custom Dashboard Integration:**
+```python
+# Example: Push metrics to your monitoring system
+import asyncio
+import websockets
+import json
+from your_monitoring import push_metric
+
+async def integrate_monitoring():
+ async with websockets.connect("ws://localhost:11235/monitor/ws") as ws:
+ while True:
+ data = json.loads(await ws.recv())
+
+ # Push to your monitoring system
+ push_metric("crawl4ai.memory.percent",
+ data['health']['container']['memory_percent'])
+ push_metric("crawl4ai.active_requests",
+ len(data['requests']['active']))
+ push_metric("crawl4ai.browser_count",
+ len(data['browsers']))
+```
+
+**Alerting Example:**
+```python
+import requests
+import time
+
+def check_health():
+ """Poll health endpoint and alert on issues"""
+ response = requests.get("http://localhost:11235/monitor/health")
+ health = response.json()
+
+ # Alert on high memory
+ if health['container']['memory_percent'] > 85:
+ send_alert(f"High memory: {health['container']['memory_percent']}%")
+
+ # Alert on high error rate
+ stats = requests.get("http://localhost:11235/monitor/endpoints/stats").json()
+ for endpoint, metrics in stats.items():
+ if metrics['success_rate_percent'] < 95:
+ send_alert(f"{endpoint} success rate: {metrics['success_rate_percent']}%")
+
+# Run every minute
+while True:
+ check_health()
+ time.sleep(60)
+```
+
+**Log Aggregation:**
+```python
+import requests
+from datetime import datetime
+
+def aggregate_errors():
+ """Fetch and aggregate errors for logging system"""
+ response = requests.get("http://localhost:11235/monitor/logs/errors?limit=100")
+ errors = response.json()['errors']
+
+ for error in errors:
+ log_to_system({
+ 'timestamp': datetime.fromtimestamp(error['timestamp']),
+ 'service': 'crawl4ai',
+ 'endpoint': error['endpoint'],
+ 'url': error['url'],
+ 'message': error['error'],
+ 'request_id': error['request_id']
+ })
+```
+
+#### Key Metrics to Track
+
+For production self-hosted deployments, monitor these metrics:
+
+1. **Memory Usage Trends**
+ - Track `container.memory_percent` over time
+ - Alert when consistently above 80%
+ - Prevents OOM kills
+
+2. **Request Success Rates**
+ - Monitor per-endpoint success rates
+ - Alert when below 95%
+ - Indicates crawling issues
+
+3. **Average Latency**
+ - Track `avg_latency_ms` per endpoint
+ - Detect performance degradation
+ - Optimize slow endpoints
+
+4. **Browser Pool Efficiency**
+ - Monitor `reuse_rate_percent`
+ - Should be >80% for good efficiency
+ - Low rates indicate pool churn
+
+5. **Error Frequency**
+ - Count errors per time window
+ - Alert on sudden spikes
+ - Track error patterns
+
+6. **Janitor Activity**
+ - Monitor cleanup frequency
+ - Excessive cleanup indicates memory pressure
+ - Adjust pool settings if needed
+
+---
+
+### Quick Health Check
+
+For simple uptime monitoring:
+
+```bash
+curl http://localhost:11235/health
+```
+
+Returns:
+```json
+{
+ "status": "healthy",
+ "version": "0.7.4"
+}
+```
+
+Other useful endpoints:
+- `/metrics` - Prometheus metrics
+- `/schema` - Full API schema
---
@@ -2132,43 +2573,46 @@ We're here to help you succeed with Crawl4AI! Here's how to get support:
## Summary
-In this guide, we've covered everything you need to get started with Crawl4AI's Docker deployment:
-- Building and running the Docker container
-- Configuring the environment
-- Using the interactive playground for testing
-- Making API requests with proper typing
-- Using the Python SDK with **automatic hook conversion**
-- **Working with hooks** - both string-based (REST API) and function-based (Python SDK)
-- Leveraging specialized endpoints for screenshots, PDFs, and JavaScript execution
-- Connecting via the Model Context Protocol (MCP)
-- Monitoring your deployment
+Congratulations! You now have everything you need to self-host your own Crawl4AI infrastructure with complete control and visibility.
-### Key Features
+**What You've Learned:**
+- ✅ Multiple deployment options (Docker Hub, Docker Compose, manual builds)
+- ✅ Environment configuration and LLM integration
+- ✅ Using the interactive playground for testing
+- ✅ Making API requests with proper typing (SDK and REST)
+- ✅ Specialized endpoints (screenshots, PDFs, JavaScript execution)
+- ✅ MCP integration for AI-assisted development
+- ✅ **Real-time monitoring dashboard** for operational transparency
+- ✅ **Monitor API** for programmatic control and integration
+- ✅ Production deployment best practices
-**Hooks Support**: Crawl4AI offers two approaches for working with hooks:
-- **String-based** (REST API): Works with any language, requires manual string formatting
-- **Function-based** (Python SDK): Write hooks as regular Python functions with full IDE support and automatic conversion
+**Why This Matters:**
-**Playground Interface**: The built-in playground at `http://localhost:11235/playground` makes it easy to test configurations and generate corresponding JSON for API requests.
+By self-hosting Crawl4AI, you:
+- 🔒 **Own Your Data**: Everything stays in your infrastructure
+- 📊 **See Everything**: Real-time dashboard shows exactly what's happening
+- 💰 **Control Costs**: Scale within your resources, no per-request fees
+- ⚡ **Maximize Performance**: Direct access with smart browser pooling (10x memory efficiency)
+- 🛡️ **Stay Secure**: Keep sensitive workflows behind your firewall
+- 🔧 **Customize Freely**: Full control over configs, strategies, and optimizations
-**MCP Integration**: For AI application developers, the MCP integration allows tools like Claude Code to directly access Crawl4AI's capabilities without complex API handling.
+**Next Steps:**
-### Next Steps
+1. **Start Simple**: Deploy with Docker Hub image and test with the playground
+2. **Monitor Everything**: Open `http://localhost:11235/monitor` to watch your server
+3. **Integrate**: Connect your applications using the Python SDK or REST API
+4. **Scale Smart**: Use the monitoring data to optimize your deployment
+5. **Go Production**: Set up alerting, log aggregation, and automated cleanup
-1. **Explore Examples**: Check out the comprehensive examples in:
- - `/docs/examples/hooks_docker_client_example.py` - Python function-based hooks
- - `/docs/examples/hooks_rest_api_example.py` - REST API string-based hooks
- - `/docs/examples/README_HOOKS.md` - Comparison and guide
+**Key Resources:**
+- 🎮 **Playground**: `http://localhost:11235/playground` - Interactive testing
+- 📊 **Monitor Dashboard**: `http://localhost:11235/monitor` - Real-time visibility
+- 📖 **Architecture Docs**: `deploy/docker/ARCHITECTURE.md` - Deep technical dive
+- 💬 **Discord Community**: Get help and share experiences
+- ⭐ **GitHub**: Report issues, contribute, show support
-2. **Read Documentation**:
- - `/docs/hooks-utility-guide.md` - Complete hooks utility guide
- - API documentation for detailed configuration options
+Remember: The monitoring dashboard is your window into your infrastructure. Use it to understand performance, troubleshoot issues, and optimize your deployment. The examples in the `examples` folder show real-world usage patterns you can adapt.
-3. **Join the Community**:
- - GitHub: Report issues and contribute
- - Discord: Get help and share your experiences
- - Documentation: Comprehensive guides and tutorials
-
-Keep exploring, and don't hesitate to reach out if you need help! We're building something amazing together. 🚀
+**You're now in control of your web crawling destiny!** 🚀
Happy crawling! 🕷️
diff --git a/mkdocs.yml b/mkdocs.yml
index 475fe190..ef23b455 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -19,7 +19,7 @@ nav:
- "Marketplace Admin": "marketplace/admin/index.html"
- Setup & Installation:
- "Installation": "core/installation.md"
- - "Docker Deployment": "core/docker-deployment.md"
+ - "Self-Hosting Guide": "core/self-hosting.md"
- "Blog & Changelog":
- "Blog Home": "blog/index.md"
- "Changelog": "https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md"
+ 
+