#1564: fix issue with _sig when using proxyConfig

crawl4ai/async_crawler_strategy.py

@@ -1047,28 +1047,14 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
             raise e
 
         finally:
-            # Clean up page after crawl completes
+            # If no session_id is given we should close the page
-            # For managed CDP browsers, close pages that are not part of a session to prevent memory leaks
             all_contexts = page.context.browser.contexts
-            total_pages = sum(len(context.pages) for context in all_contexts)
+            total_pages = sum(len(context.pages) for context in all_contexts)                
-            
-            should_close_page = False
-            
             if config.session_id:
-                # Session pages are kept alive for reuse
                 pass
-            elif self.browser_config.use_managed_browser:
+            elif total_pages <= 1 and (self.browser_config.use_managed_browser or self.browser_config.headless):
-                # For managed browsers (CDP), close non-session pages to prevent tab accumulation
-                # This is especially important for arun_many() with multiple concurrent crawls
-                should_close_page = True
-            elif total_pages <= 1 and self.browser_config.headless:
-                # Keep the last page in headless mode to avoid closing the browser
                 pass
             else:
-                # For non-managed browsers, close the page
-                should_close_page = True
-            
-            if should_close_page:
                 # Detach listeners before closing to prevent potential errors during close
                 if config.capture_network_requests:
                     page.remove_listener("request", handle_request_capture)
@@ -1397,10 +1383,9 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
         try:
             await self.adapter.evaluate(page,
                 f"""
-                (async () => {{
+                (() => {{
                     try {{
-                        const removeOverlays = {remove_overlays_js};
+                        {remove_overlays_js}
-                        await removeOverlays();
                         return {{ success: true }};
                     }} catch (error) {{
                         return {{

crawl4ai/browser_manager.py

@@ -1035,20 +1035,34 @@ class BrowserManager:
             self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
             return page, context
 
-        # If using a managed browser, reuse the default context and create new pages
+        # If using a managed browser, just grab the shared default_context
         if self.config.use_managed_browser:
-            context = self.default_context
             if self.config.storage_state:
-                # Clone runtime state from storage to the shared context
+                context = await self.create_browser_context(crawlerRunConfig)
-                ctx = self.default_context
+                ctx = self.default_context        # default context, one window only
                 ctx = await clone_runtime_state(context, ctx, crawlerRunConfig, self.config)
-            
+                # Avoid concurrent new_page on shared persistent context
-            # Always create a new page for concurrent safety
+                # See GH-1198: context.pages can be empty under races
-            # The page-level isolation prevents race conditions while sharing the same context
+                async with self._page_lock:
-            async with self._page_lock:
+                    page = await ctx.new_page()
-                page = await context.new_page()
+                await self._apply_stealth_to_page(page)
-            
+            else:
-            await self._apply_stealth_to_page(page)
+                context = self.default_context
+                pages = context.pages
+                page = next((p for p in pages if p.url == crawlerRunConfig.url), None)
+                if not page:
+                    if pages:
+                        page = pages[0]
+                    else:
+                        # Double-check under lock to avoid TOCTOU and ensure only
+                        # one task calls new_page when pages=[] concurrently
+                        async with self._page_lock:
+                            pages = context.pages
+                            if pages:
+                                page = pages[0]
+                            else:
+                                page = await context.new_page()
+                                await self._apply_stealth_to_page(page)
         else:
             # Otherwise, check if we have an existing context for this config
             config_signature = self._make_config_signature(crawlerRunConfig)

deploy/docker/crawler_pool.py

@@ -2,8 +2,8 @@
 import asyncio, json, hashlib, time, psutil
 from contextlib import suppress
 from typing import Dict
-from crawl4ai import AsyncWebCrawler, BrowserConfig
+from crawl4ai import AsyncWebCrawler, BrowserConfig, BrowserAdapter
-from typing import Dict
+from typing import Dict ,Optional
 from utils import load_config 
 
 CONFIG = load_config()
@@ -15,11 +15,22 @@ LOCK = asyncio.Lock()
 MEM_LIMIT  = CONFIG.get("crawler", {}).get("memory_threshold_percent", 95.0)   # % RAM – refuse new browsers above this
 IDLE_TTL  = CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 1800)   # close if unused for 30 min
 
-def _sig(cfg: BrowserConfig) -> str:
+
-    payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",",":"))
+def _sig(cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None) -> str:
+    try:
+        config_payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",", ":"))
+    except (TypeError, ValueError):
+        # Fallback to string representation if JSON serialization fails
+        config_payload = str(cfg.to_dict())
+    adapter_name = adapter.__class__.__name__ if adapter else "PlaywrightAdapter"
+    payload = f"{config_payload}:{adapter_name}"
     return hashlib.sha1(payload.encode()).hexdigest()
 
-async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler:
+
+async def get_crawler(
+    cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None
+) -> AsyncWebCrawler:
+    sig = None
     try:
         sig = _sig(cfg)
         async with LOCK:
@@ -37,12 +48,13 @@ async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler:
     except Exception as e:
         raise RuntimeError(f"Failed to start browser: {e}")
     finally:
-        if sig in POOL:
+        if sig:
-            LAST_USED[sig] = time.time()
+            if sig in POOL:
-        else:
+                LAST_USED[sig] = time.time()
-            # If we failed to start the browser, we should remove it from the pool
+            else:
-            POOL.pop(sig, None)
+                # If we failed to start the browser, we should remove it from the pool
-            LAST_USED.pop(sig, None)
+                POOL.pop(sig, None)
+                LAST_USED.pop(sig, None)
         # If we failed to start the browser, we should remove it from the pool
 async def close_all():
     async with LOCK:

docker-compose.yml

@@ -6,16 +6,15 @@ x-base-config: &base-config
     - "11235:11235"  # Gunicorn port
   env_file:
     - .llm.env       # API keys (create from .llm.env.example)
-  # Uncomment to set default environment variables (will overwrite .llm.env)
+  environment:
-  # environment:
+    - OPENAI_API_KEY=${OPENAI_API_KEY:-}
-  #   - OPENAI_API_KEY=${OPENAI_API_KEY:-}
+    - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-}
-  #   - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-}
+    - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
-  #   - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
+    - GROQ_API_KEY=${GROQ_API_KEY:-}
-  #   - GROQ_API_KEY=${GROQ_API_KEY:-}
+    - TOGETHER_API_KEY=${TOGETHER_API_KEY:-}
-  #   - TOGETHER_API_KEY=${TOGETHER_API_KEY:-}
+    - MISTRAL_API_KEY=${MISTRAL_API_KEY:-}
-  #   - MISTRAL_API_KEY=${MISTRAL_API_KEY:-}
+    - GEMINI_API_TOKEN=${GEMINI_API_TOKEN:-}
-  #   - GEMINI_API_KEY=${GEMINI_API_KEY:-}
+    - LLM_PROVIDER=${LLM_PROVIDER:-}  # Optional: Override default provider (e.g., "anthropic/claude-3-opus")
-  #   - LLM_PROVIDER=${LLM_PROVIDER:-}  # Optional: Override default provider (e.g., "anthropic/claude-3-opus")
   volumes:
     - /dev/shm:/dev/shm  # Chromium performance
   deploy:

docs/examples/c4a_script/tutorial/README.md

@@ -18,7 +18,7 @@ A comprehensive web-based tutorial for learning and experimenting with C4A-Scrip
 
 2. **Install Dependencies**
    ```bash
-   pip install -r requirements.txt
+   pip install flask
    ```
 
 3. **Launch the Server**
@@ -28,7 +28,7 @@ A comprehensive web-based tutorial for learning and experimenting with C4A-Scrip
 
 4. **Open in Browser**
    ```
-   http://localhost:8000
+   http://localhost:8080
    ```
 
 **🌐 Try Online**: [Live Demo](https://docs.crawl4ai.com/c4a-script/demo)
@@ -325,7 +325,7 @@ Powers the recording functionality:
 ### Configuration
 ```python
 # server.py configuration
-PORT = 8000
+PORT = 8080
 DEBUG = True
 THREADED = True
 ```
@@ -343,9 +343,9 @@ THREADED = True
 **Port Already in Use**
 ```bash
 # Kill existing process
-lsof -ti:8000 | xargs kill -9
+lsof -ti:8080 | xargs kill -9
 # Or use different port
-python server.py --port 8001
+python server.py --port 8081
 ```
 
 **Blockly Not Loading**

docs/examples/c4a_script/tutorial/server.py

@@ -216,7 +216,7 @@ def get_examples():
             'name': 'Handle Cookie Banner',
             'description': 'Accept cookies and close newsletter popup',
             'script': '''# Handle cookie banner and newsletter
-GO http://127.0.0.1:8000/playground/
+GO http://127.0.0.1:8080/playground/
 WAIT `body` 2
 IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
 IF (EXISTS `.newsletter-popup`) THEN CLICK `.close`'''

docs/md_v2/advanced/cdp-browser-crawling.md

@@ -1,594 +0,0 @@
-# CDP Browser Crawling
-
-> **New in v0.7.6**: Efficient concurrent crawling with managed CDP (Chrome DevTools Protocol) browsers. Connect to a running browser instance and perform multiple crawls without spawning new windows.
-
-## 1. Overview
-
-When working with CDP browsers, you can connect to an existing browser instance instead of launching a new one for each crawl. This is particularly useful for:
-
-- **Development**: Keep your browser open with DevTools for debugging
-- **Persistent Sessions**: Maintain authentication across multiple crawls  
-- **Resource Efficiency**: Reuse a single browser instance for multiple operations
-- **Concurrent Crawling**: Run multiple crawls simultaneously with proper isolation
-
-**Key Benefits:**
-
-- ✅ Single browser window with multiple tabs (no window clutter)
-- ✅ Shared state (cookies, localStorage) across crawls
-- ✅ Concurrent safety with automatic page isolation
-- ✅ Automatic cleanup to prevent memory leaks
-- ✅ Works seamlessly with `arun_many()` for parallel crawling
-
----
-
-## 2. Quick Start
-
-### 2.1 Starting a CDP Browser
-
-Use the Crawl4AI CLI to start a managed CDP browser:
-
-```bash
-# Start CDP browser on default port (9222)
-crwl cdp
-
-# Start on custom port
-crwl cdp -d 9223
-
-# Start in headless mode
-crwl cdp --headless
-```
-
-The browser will stay running until you press 'q' or close the terminal.
-
-### 2.2 Basic CDP Connection
-
-```python
-import asyncio
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
-
-async def main():
-    # Configure CDP connection
-    browser_cfg = BrowserConfig(
-        browser_type="chromium",
-        cdp_url="http://localhost:9222",
-        verbose=True
-    )
-    
-    # Crawl a single URL
-    async with AsyncWebCrawler(config=browser_cfg) as crawler:
-        result = await crawler.arun(
-            url="https://example.com",
-            config=CrawlerRunConfig()
-        )
-        print(f"Success: {result.success}")
-        print(f"Content length: {len(result.markdown)}")
-
-if __name__ == "__main__":
-    asyncio.run(main())
-```
-
----
-
-## 3. Concurrent Crawling with arun_many()
-
-The real power of CDP crawling shines with `arun_many()`. The browser manager automatically handles:
-
-- **Page Isolation**: Each crawl gets its own tab
-- **Context Sharing**: All tabs share cookies and localStorage
-- **Concurrent Safety**: Proper locking prevents race conditions
-- **Auto Cleanup**: Tabs are closed after crawling (except sessions)
-
-### 3.1 Basic Concurrent Crawling
-
-```python
-import asyncio
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
-
-async def crawl_multiple_urls():
-    # URLs to crawl
-    urls = [
-        "https://example.com",
-        "https://httpbin.org/html",
-        "https://www.python.org",
-    ]
-    
-    # Configure CDP browser
-    browser_cfg = BrowserConfig(
-        browser_type="chromium",
-        cdp_url="http://localhost:9222",
-        verbose=False
-    )
-    
-    # Configure crawler (bypass cache for fresh data)
-    crawler_cfg = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS
-    )
-    
-    # Crawl all URLs concurrently
-    async with AsyncWebCrawler(config=browser_cfg) as crawler:
-        results = await crawler.arun_many(
-            urls=urls,
-            config=crawler_cfg
-        )
-        
-        # Process results
-        for result in results:
-            print(f"\nURL: {result.url}")
-            if result.success:
-                print(f"✓ Success | Content length: {len(result.markdown)}")
-            else:
-                print(f"✗ Failed: {result.error_message}")
-
-if __name__ == "__main__":
-    asyncio.run(crawl_multiple_urls())
-```
-
-### 3.2 With Session Management
-
-Use sessions to maintain authentication and state across individual crawls:
-
-```python
-async def crawl_with_sessions():
-    browser_cfg = BrowserConfig(
-        browser_type="chromium",
-        cdp_url="http://localhost:9222"
-    )
-    
-    async with AsyncWebCrawler(config=browser_cfg) as crawler:
-        # First crawl: Login page
-        login_result = await crawler.arun(
-            url="https://example.com/login",
-            config=CrawlerRunConfig(
-                session_id="my-session",  # Session persists
-                js_code="document.querySelector('#login').click();"
-            )
-        )
-        
-        # Second crawl: Reuse authenticated session
-        dashboard_result = await crawler.arun(
-            url="https://example.com/dashboard",
-            config=CrawlerRunConfig(
-                session_id="my-session"  # Same session, cookies preserved
-            )
-        )
-```
-
----
-
-## 4. How It Works
-
-### 4.1 Browser Context Reuse
-
-When using CDP browsers, Crawl4AI:
-
-1. **Connects** to the existing browser via CDP URL
-2. **Reuses** the default browser context (single window)
-3. **Creates** new pages (tabs) for each crawl
-4. **Locks** page creation to prevent concurrent races
-5. **Cleans up** pages after crawling (unless it's a session)
-
-```python
-# Internal behavior (simplified)
-if self.config.use_managed_browser:
-    context = self.default_context  # Shared context
-    
-    # Thread-safe page creation
-    async with self._page_lock:
-        page = await context.new_page()  # New tab per crawl
-    
-    # After crawl completes
-    if not config.session_id:
-        await page.close()  # Auto cleanup
-```
-
-### 4.2 Page Lifecycle
-
-```mermaid
-graph TD
-    A[Start Crawl] --> B{Has session_id?}
-    B -->|Yes| C[Reuse existing page]
-    B -->|No| D[Create new page/tab]
-    D --> E[Navigate & Extract]
-    C --> E
-    E --> F{Is session?}
-    F -->|Yes| G[Keep page open]
-    F -->|No| H[Close page]
-    H --> I[End]
-    G --> I
-```
-
-### 4.3 State Sharing
-
-All pages in the same context share:
-
-- 🍪 **Cookies**: Authentication tokens, preferences
-- 💾 **localStorage**: Client-side data storage
-- 🔐 **sessionStorage**: Per-tab session data
-- 🌐 **Network cache**: Shared HTTP cache
-
-This makes it perfect for crawling authenticated sites or maintaining state across multiple pages.
-
----
-
-## 5. Configuration Options
-
-### 5.1 BrowserConfig for CDP
-
-```python
-browser_cfg = BrowserConfig(
-    browser_type="chromium",           # Must be "chromium" for CDP
-    cdp_url="http://localhost:9222",   # CDP endpoint URL
-    verbose=True,                      # Log browser operations
-    
-    # Optional: Override headers for all requests
-    headers={
-        "Accept-Language": "en-US,en;q=0.9",
-    },
-    
-    # Optional: Set user agent
-    user_agent="Mozilla/5.0 ...",
-    
-    # Optional: Enable stealth mode (requires dedicated browser)
-    # enable_stealth=False,  # Not compatible with CDP
-)
-```
-
-### 5.2 CrawlerRunConfig Options
-
-```python
-crawler_cfg = CrawlerRunConfig(
-    # Session management
-    session_id="my-session",           # Persist page across calls
-    
-    # Caching
-    cache_mode=CacheMode.BYPASS,       # Fresh data every time
-    
-    # Browser location (affects timezone, locale)
-    locale="en-US",
-    timezone_id="America/New_York",
-    geolocation={
-        "latitude": 40.7128,
-        "longitude": -74.0060
-    },
-    
-    # Proxy (per-crawl override)
-    proxy_config={
-        "server": "http://proxy.example.com:8080",
-        "username": "user",
-        "password": "pass"
-    }
-)
-```
-
----
-
-## 6. Advanced Patterns
-
-### 6.1 Streaming Results
-
-Process URLs as they complete instead of waiting for all:
-
-```python
-async def stream_crawl_results():
-    browser_cfg = BrowserConfig(
-        browser_type="chromium",
-        cdp_url="http://localhost:9222"
-    )
-    
-    urls = ["https://example.com" for _ in range(100)]
-    
-    async with AsyncWebCrawler(config=browser_cfg) as crawler:
-        # Stream results as they complete
-        async for result in crawler.arun_many(
-            urls=urls,
-            config=CrawlerRunConfig(stream=True)
-        ):
-            if result.success:
-                print(f"✓ {result.url}: {len(result.markdown)} chars")
-                # Process immediately instead of waiting for all
-                await save_to_database(result)
-```
-
-### 6.2 Custom Concurrency Control
-
-```python
-from crawl4ai import CrawlerRunConfig
-
-# Limit concurrent crawls to 3
-crawler_cfg = CrawlerRunConfig(
-    semaphore_count=3,      # Max 3 concurrent requests
-    mean_delay=0.5,         # Average 0.5s delay between requests
-    max_range=1.0,          # +/- 1s random delay
-)
-
-async with AsyncWebCrawler(config=browser_cfg) as crawler:
-    results = await crawler.arun_many(urls, config=crawler_cfg)
-```
-
-### 6.3 Multi-Config Crawling
-
-Different configurations for different URL groups:
-
-```python
-from crawl4ai import CrawlerRunConfig
-
-# Fast crawl for static pages
-fast_config = CrawlerRunConfig(
-    wait_until="domcontentloaded",
-    page_timeout=30000
-)
-
-# Slow crawl for dynamic pages
-slow_config = CrawlerRunConfig(
-    wait_until="networkidle",
-    page_timeout=60000,
-    js_code="window.scrollTo(0, document.body.scrollHeight);"
-)
-
-configs = [fast_config, slow_config, fast_config]
-urls = ["https://static.com", "https://dynamic.com", "https://static2.com"]
-
-async with AsyncWebCrawler(config=browser_cfg) as crawler:
-    results = await crawler.arun_many(urls, configs=configs)
-```
-
----
-
-## 7. Best Practices
-
-### 7.1 Resource Management
-
-✅ **DO:**
-```python
-# Use context manager for automatic cleanup
-async with AsyncWebCrawler(config=browser_cfg) as crawler:
-    results = await crawler.arun_many(urls)
-# Browser connection closed automatically
-```
-
-❌ **DON'T:**
-```python
-# Manual management risks resource leaks
-crawler = AsyncWebCrawler(config=browser_cfg)
-await crawler.start()
-results = await crawler.arun_many(urls)
-# Forgot to call crawler.close()!
-```
-
-### 7.2 Session Management
-
-✅ **DO:**
-```python
-# Use sessions for related crawls
-config = CrawlerRunConfig(session_id="user-flow")
-await crawler.arun(login_url, config=config)
-await crawler.arun(dashboard_url, config=config)
-await crawler.kill_session("user-flow")  # Clean up when done
-```
-
-❌ **DON'T:**
-```python
-# Creating new session IDs unnecessarily
-for i in range(100):
-    config = CrawlerRunConfig(session_id=f"session-{i}")
-    await crawler.arun(url, config=config)
-# 100 unclosed sessions accumulate!
-```
-
-### 7.3 Error Handling
-
-```python
-async def robust_crawl(urls):
-    browser_cfg = BrowserConfig(
-        browser_type="chromium",
-        cdp_url="http://localhost:9222"
-    )
-    
-    try:
-        async with AsyncWebCrawler(config=browser_cfg) as crawler:
-            results = await crawler.arun_many(urls)
-            
-            # Separate successes and failures
-            successes = [r for r in results if r.success]
-            failures = [r for r in results if not r.success]
-            
-            print(f"✓ {len(successes)} succeeded")
-            print(f"✗ {len(failures)} failed")
-            
-            # Retry failures with different config
-            if failures:
-                retry_urls = [r.url for r in failures]
-                retry_config = CrawlerRunConfig(
-                    page_timeout=120000,  # Longer timeout
-                    wait_until="networkidle"
-                )
-                retry_results = await crawler.arun_many(
-                    retry_urls, 
-                    config=retry_config
-                )
-                
-            return successes + retry_results
-            
-    except Exception as e:
-        print(f"Fatal error: {e}")
-        return []
-```
-
----
-
-## 8. Troubleshooting
-
-### 8.1 Connection Issues
-
-**Problem**: `Cannot connect to CDP browser`
-
-```python
-# Check CDP browser is running
-$ lsof -i :9222
-# Should show: Chromium  PID USER   FD   TYPE  ...
-
-# Or start it if not running
-$ crwl cdp
-```
-
-**Problem**: `ERR_ABORTED` errors in concurrent crawls
-
-✅ **Fixed in v0.7.6**: This issue has been resolved. Pages are now properly isolated with locking.
-
-### 8.2 Performance Issues
-
-**Problem**: Too many open tabs
-
-```python
-# Ensure you're not using session_id for everything
-config = CrawlerRunConfig()  # No session_id
-await crawler.arun_many(urls, config=config)
-# Pages auto-close after crawling
-```
-
-**Problem**: Memory leaks
-
-```python
-# Always use context manager
-async with AsyncWebCrawler(config=browser_cfg) as crawler:
-    # Crawling code here
-    pass
-# Automatic cleanup on exit
-```
-
-### 8.3 State Issues
-
-**Problem**: Cookies not persisting
-
-```python
-# Use the same context (automatic with CDP)
-browser_cfg = BrowserConfig(cdp_url="http://localhost:9222")
-# All crawls share cookies automatically
-```
-
-**Problem**: Need isolated state
-
-```python
-# Use different CDP endpoints or non-CDP browsers
-browser_cfg_1 = BrowserConfig(cdp_url="http://localhost:9222")
-browser_cfg_2 = BrowserConfig(cdp_url="http://localhost:9223")
-# Completely isolated browsers
-```
-
----
-
-## 9. Comparison: CDP vs Regular Browsers
-
-| Feature | CDP Browser | Regular Browser |
-|---------|-------------|-----------------|
-| **Window Management** | ✅ Single window, multiple tabs | ❌ New window per context |
-| **Startup Time** | ✅ Instant (already running) | ⏱️ ~2-3s per launch |
-| **State Sharing** | ✅ Shared cookies/localStorage | ⚠️ Isolated by default |
-| **Concurrent Safety** | ✅ Automatic locking | ✅ Separate processes |
-| **Memory Usage** | ✅ Lower (shared browser) | ⚠️ Higher (multiple processes) |
-| **Session Persistence** | ✅ Native support | ✅ Via session_id |
-| **Stealth Mode** | ❌ Not compatible | ✅ Full support |
-| **Best For** | Development, authenticated crawls | Production, isolated crawls |
-
----
-
-## 10. Real-World Examples
-
-### 10.1 E-commerce Product Scraping
-
-```python
-async def scrape_products():
-    browser_cfg = BrowserConfig(
-        browser_type="chromium",
-        cdp_url="http://localhost:9222"
-    )
-    
-    # Get product URLs from category page
-    async with AsyncWebCrawler(config=browser_cfg) as crawler:
-        category_result = await crawler.arun(
-            url="https://shop.example.com/category",
-            config=CrawlerRunConfig(
-                css_selector=".product-link"
-            )
-        )
-        
-        # Extract product URLs
-        product_urls = extract_urls(category_result.links)
-        
-        # Crawl all products concurrently
-        product_results = await crawler.arun_many(
-            urls=product_urls,
-            config=CrawlerRunConfig(
-                css_selector=".product-details",
-                semaphore_count=5  # Polite crawling
-            )
-        )
-        
-        return [extract_product_data(r) for r in product_results]
-```
-
-### 10.2 News Article Monitoring
-
-```python
-import asyncio
-from datetime import datetime
-
-async def monitor_news_sites():
-    browser_cfg = BrowserConfig(
-        browser_type="chromium",
-        cdp_url="http://localhost:9222"
-    )
-    
-    news_sites = [
-        "https://news.site1.com",
-        "https://news.site2.com",
-        "https://news.site3.com"
-    ]
-    
-    async with AsyncWebCrawler(config=browser_cfg) as crawler:
-        while True:
-            print(f"\n[{datetime.now()}] Checking for updates...")
-            
-            results = await crawler.arun_many(
-                urls=news_sites,
-                config=CrawlerRunConfig(
-                    cache_mode=CacheMode.BYPASS,  # Always fresh
-                    css_selector=".article-headline"
-                )
-            )
-            
-            for result in results:
-                if result.success:
-                    headlines = extract_headlines(result)
-                    for headline in headlines:
-                        if is_new(headline):
-                            notify_user(headline)
-            
-            # Check every 5 minutes
-            await asyncio.sleep(300)
-```
-
----
-
-## 11. Summary
-
-CDP browser crawling offers:
-
-- 🚀 **Performance**: Faster startup, lower resource usage
-- 🔄 **State Management**: Shared cookies and authentication
-- 🎯 **Concurrent Safety**: Automatic page isolation and cleanup
-- 💻 **Developer Friendly**: Visual debugging with DevTools
-
-**When to use CDP:**
-- Development and debugging
-- Authenticated crawling (login required)
-- Sequential crawls needing state
-- Resource-constrained environments
-
-**When to use regular browsers:**
-- Production deployments
-- Maximum isolation required  
-- Stealth mode needed
-- Distributed/cloud crawling
-
-For most use cases, **CDP browsers provide the best balance** of performance, convenience, and safety.

docs/md_v2/advanced/identity-based-crawling.md

@@ -82,42 +82,6 @@ If you installed Crawl4AI (which installs Playwright under the hood), you alread
 
 ---
 
-### Creating a Profile Using the Crawl4AI CLI (Easiest)
-
-If you prefer a guided, interactive setup, use the built-in CLI to create and manage persistent browser profiles.
-
-1.⠀Launch the profile manager:
-   ```bash
-   crwl profiles
-   ```
-
-2.⠀Choose "Create new profile" and enter a profile name. A Chromium window opens so you can log in to sites and configure settings. When finished, return to the terminal and press `q` to save the profile.
-
-3.⠀Profiles are saved under `~/.crawl4ai/profiles/<profile_name>` (for example: `/home/<you>/.crawl4ai/profiles/test_profile_1`) along with a `storage_state.json` for cookies and session data.
-
-4.⠀Optionally, choose "List profiles" in the CLI to view available profiles and their paths.
-
-5.⠀Use the saved path with `BrowserConfig.user_data_dir`:
-   ```python
-   from crawl4ai import AsyncWebCrawler, BrowserConfig
-
-   profile_path = "/home/<you>/.crawl4ai/profiles/test_profile_1"
-
-   browser_config = BrowserConfig(
-       headless=True,
-       use_managed_browser=True,
-       user_data_dir=profile_path,
-       browser_type="chromium",
-   )
-
-   async with AsyncWebCrawler(config=browser_config) as crawler:
-       result = await crawler.arun(url="https://example.com/private")
-   ```
-
-The CLI also supports listing and deleting profiles, and even testing a crawl directly from the menu.
-
----
-
 ## 3. Using Managed Browsers in Crawl4AI
 
 Once you have a data directory with your session data, pass it to **`BrowserConfig`**:

docs/md_v2/apps/c4a-script/README.md

@@ -18,7 +18,7 @@ A comprehensive web-based tutorial for learning and experimenting with C4A-Scrip
 
 2. **Install Dependencies**
    ```bash
-   pip install -r requirements.txt
+   pip install flask
    ```
 
 3. **Launch the Server**
@@ -28,7 +28,7 @@ A comprehensive web-based tutorial for learning and experimenting with C4A-Scrip
 
 4. **Open in Browser**
    ```
-   http://localhost:8000
+   http://localhost:8080
    ```
 
 **🌐 Try Online**: [Live Demo](https://docs.crawl4ai.com/c4a-script/demo)
@@ -325,7 +325,7 @@ Powers the recording functionality:
 ### Configuration
 ```python
 # server.py configuration
-PORT = 8000
+PORT = 8080
 DEBUG = True
 THREADED = True
 ```
@@ -343,9 +343,9 @@ THREADED = True
 **Port Already in Use**
 ```bash
 # Kill existing process
-lsof -ti:8000 | xargs kill -9
+lsof -ti:8080 | xargs kill -9
 # Or use different port
-python server.py --port 8001
+python server.py --port 8081
 ```
 
 **Blockly Not Loading**

docs/md_v2/apps/c4a-script/server.py

@@ -216,7 +216,7 @@ def get_examples():
             'name': 'Handle Cookie Banner',
             'description': 'Accept cookies and close newsletter popup',
             'script': '''# Handle cookie banner and newsletter
-GO http://127.0.0.1:8000/playground/
+GO http://127.0.0.1:8080/playground/
 WAIT `body` 2
 IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
 IF (EXISTS `.newsletter-popup`) THEN CLICK `.close`'''
@@ -283,7 +283,7 @@ WAIT `.success-message` 5'''
     return jsonify(examples)
 
 if __name__ == '__main__':
-    port = int(os.environ.get('PORT', 8000))
+    port = int(os.environ.get('PORT', 8080))
     print(f"""
 ╔══════════════════════════════════════════════════════════╗
 ║          C4A-Script Interactive Tutorial Server          ║

docs/md_v2/core/c4a-script.md

@@ -69,12 +69,12 @@ The tutorial includes a Flask-based web interface with:
 cd docs/examples/c4a_script/tutorial/
 
 # Install dependencies
-pip install -r requirements.txt
+pip install flask
 
 # Launch the tutorial server
-python server.py
+python app.py
 
-# Open http://localhost:8000 in your browser
+# Open http://localhost:5000 in your browser
 ```
 
 ## Core Concepts
@@ -111,8 +111,8 @@ CLICK `.submit-btn`
 # By attribute
 CLICK `button[type="submit"]`
 
-# By accessible attributes
+# By text content
-CLICK `button[aria-label="Search"][title="Search"]`
+CLICK `button:contains("Sign In")`
 
 # Complex selectors
 CLICK `.form-container input[name="email"]`

docs/md_v2/index.md

@@ -57,7 +57,7 @@
 
 Crawl4AI is the #1 trending GitHub repository, actively maintained by a vibrant community. It delivers blazing-fast, AI-ready web crawling tailored for large language models, AI agents, and data pipelines. Fully open source, flexible, and built for real-time performance, **Crawl4AI** empowers developers with unmatched speed, precision, and deployment ease.
 
-> Enjoy using Crawl4AI? Consider **[becoming a sponsor](https://github.com/sponsors/unclecode)** to support ongoing development and community growth!
+> **Note**: If you're looking for the old documentation, you can access it [here](https://old.docs.crawl4ai.com).
 
 ## 🆕 AI Assistant Skill Now Available!
 

pyproject.toml

@@ -31,7 +31,7 @@ dependencies = [
     "rank-bm25~=0.2",
     "snowballstemmer~=2.2",
     "pydantic>=2.10",
-    "pyOpenSSL>=25.3.0",
+    "pyOpenSSL>=24.3.0",
     "psutil>=6.1.1",
     "PyYAML>=6.0",
     "nltk>=3.9.1",

requirements.txt

@@ -19,7 +19,7 @@ rank-bm25~=0.2
 colorama~=0.4
 snowballstemmer~=2.2
 pydantic>=2.10
-pyOpenSSL>=25.3.0
+pyOpenSSL>=24.3.0
 psutil>=6.1.1
 PyYAML>=6.0
 nltk>=3.9.1

tests/general/test_async_crawler_strategy.py

@@ -364,19 +364,5 @@ async def test_network_error_handling():
         async with AsyncPlaywrightCrawlerStrategy() as strategy:
             await strategy.crawl("https://invalid.example.com", config)
 
-@pytest.mark.asyncio
-async def test_remove_overlay_elements(crawler_strategy):
-    config = CrawlerRunConfig(
-        remove_overlay_elements=True,
-        delay_before_return_html=5,
-    )
-    
-    response = await crawler_strategy.crawl(
-        "https://www2.hm.com/en_us/index.html",
-        config
-    )
-    assert response.status_code == 200
-    assert "Accept all cookies" not in response.html
-
 if __name__ == "__main__":
     pytest.main([__file__, "-v"])

tests/test_arun_many_cdp.py

@@ -1,63 +0,0 @@
-"""
-Test for arun_many with managed CDP browser to ensure each crawl gets its own tab.
-"""
-import pytest
-import asyncio
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
-
-
-@pytest.mark.asyncio
-async def test_arun_many_with_cdp():
-    """Test arun_many opens a new tab for each url with managed CDP browser."""
-    # NOTE: Requires a running CDP browser at localhost:9222
-    # Can be started with: crwl cdp -d 9222
-    browser_cfg = BrowserConfig(
-        browser_type="cdp",
-        cdp_url="http://localhost:9222",
-        verbose=False,
-    )
-    urls = [
-        "https://example.com",
-        "https://httpbin.org/html",
-        "https://www.python.org",
-    ]
-    crawler_cfg = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-    )
-    async with AsyncWebCrawler(config=browser_cfg) as crawler:
-        results = await crawler.arun_many(urls=urls, config=crawler_cfg)
-        # All results should be successful and distinct
-        assert len(results) == 3
-        for result in results:
-            assert result.success, f"Crawl failed: {result.url} - {result.error_message}"
-            assert result.markdown is not None
-
-
-@pytest.mark.asyncio
-async def test_arun_many_with_cdp_sequential():
-    """Test arun_many sequentially to isolate issues."""
-    browser_cfg = BrowserConfig(
-        browser_type="cdp",
-        cdp_url="http://localhost:9222",
-        verbose=True,
-    )
-    urls = [
-        "https://example.com",
-        "https://httpbin.org/html", 
-        "https://www.python.org",
-    ]
-    crawler_cfg = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-    )
-    async with AsyncWebCrawler(config=browser_cfg) as crawler:
-        results = []
-        for url in urls:
-            result = await crawler.arun(url=url, config=crawler_cfg)
-            results.append(result)
-            assert result.success, f"Crawl failed: {result.url} - {result.error_message}"
-            assert result.markdown is not None
-        assert len(results) == 3
-
-
-if __name__ == "__main__":
-    asyncio.run(test_arun_many_with_cdp())

tests/test_pyopenssl_security_fix.py

@@ -1,168 +0,0 @@
-"""
-Lightweight test to verify pyOpenSSL security fix (Issue #1545).
-
-This test verifies the security requirements are met:
-1. pyOpenSSL >= 25.3.0 is installed
-2. cryptography >= 45.0.7 is installed (above vulnerable range)
-3. SSL/TLS functionality works correctly
-
-This test can run without full crawl4ai dependencies installed.
-"""
-
-import sys
-from packaging import version
-
-
-def test_package_versions():
-    """Test that package versions meet security requirements."""
-    print("=" * 70)
-    print("TEST: Package Version Security Requirements (Issue #1545)")
-    print("=" * 70)
-
-    all_passed = True
-
-    # Test pyOpenSSL version
-    try:
-        import OpenSSL
-        pyopenssl_version = OpenSSL.__version__
-        print(f"\n✓ pyOpenSSL is installed: {pyopenssl_version}")
-
-        if version.parse(pyopenssl_version) >= version.parse("25.3.0"):
-            print(f"  ✓ PASS: pyOpenSSL {pyopenssl_version} >= 25.3.0 (required)")
-        else:
-            print(f"  ✗ FAIL: pyOpenSSL {pyopenssl_version} < 25.3.0 (required)")
-            all_passed = False
-
-    except ImportError as e:
-        print(f"\n✗ FAIL: pyOpenSSL not installed - {e}")
-        all_passed = False
-
-    # Test cryptography version
-    try:
-        import cryptography
-        crypto_version = cryptography.__version__
-        print(f"\n✓ cryptography is installed: {crypto_version}")
-
-        # The vulnerable range is >=37.0.0 & <43.0.1
-        # We need >= 45.0.7 to be safe
-        if version.parse(crypto_version) >= version.parse("45.0.7"):
-            print(f"  ✓ PASS: cryptography {crypto_version} >= 45.0.7 (secure)")
-            print(f"  ✓ NOT in vulnerable range (37.0.0 to 43.0.0)")
-        elif version.parse(crypto_version) >= version.parse("37.0.0") and version.parse(crypto_version) < version.parse("43.0.1"):
-            print(f"  ✗ FAIL: cryptography {crypto_version} is VULNERABLE")
-            print(f"  ✗ Version is in vulnerable range (>=37.0.0 & <43.0.1)")
-            all_passed = False
-        else:
-            print(f"  ⚠ WARNING: cryptography {crypto_version} < 45.0.7")
-            print(f"  ⚠ May not meet security requirements")
-
-    except ImportError as e:
-        print(f"\n✗ FAIL: cryptography not installed - {e}")
-        all_passed = False
-
-    return all_passed
-
-
-def test_ssl_basic_functionality():
-    """Test that SSL/TLS basic functionality works."""
-    print("\n" + "=" * 70)
-    print("TEST: SSL/TLS Basic Functionality")
-    print("=" * 70)
-
-    try:
-        import OpenSSL.SSL
-
-        # Create a basic SSL context to verify functionality
-        context = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_2_METHOD)
-        print("\n✓ SSL Context created successfully")
-        print("  ✓ PASS: SSL/TLS functionality is working")
-        return True
-
-    except Exception as e:
-        print(f"\n✗ FAIL: SSL functionality test failed - {e}")
-        return False
-
-
-def test_pyopenssl_crypto_integration():
-    """Test that pyOpenSSL and cryptography integration works."""
-    print("\n" + "=" * 70)
-    print("TEST: pyOpenSSL <-> cryptography Integration")
-    print("=" * 70)
-
-    try:
-        from OpenSSL import crypto
-
-        # Generate a simple key pair to test integration
-        key = crypto.PKey()
-        key.generate_key(crypto.TYPE_RSA, 2048)
-
-        print("\n✓ Generated RSA key pair successfully")
-        print("  ✓ PASS: pyOpenSSL and cryptography are properly integrated")
-        return True
-
-    except Exception as e:
-        print(f"\n✗ FAIL: Integration test failed - {e}")
-        import traceback
-        traceback.print_exc()
-        return False
-
-
-def main():
-    """Run all security tests."""
-    print("\n")
-    print("╔" + "=" * 68 + "╗")
-    print("║  pyOpenSSL Security Fix Verification - Issue #1545               ║")
-    print("╚" + "=" * 68 + "╝")
-    print("\nVerifying that the pyOpenSSL update resolves the security vulnerability")
-    print("in the cryptography package (CVE: versions >=37.0.0 & <43.0.1)\n")
-
-    results = []
-
-    # Test 1: Package versions
-    results.append(("Package Versions", test_package_versions()))
-
-    # Test 2: SSL functionality
-    results.append(("SSL Functionality", test_ssl_basic_functionality()))
-
-    # Test 3: Integration
-    results.append(("pyOpenSSL-crypto Integration", test_pyopenssl_crypto_integration()))
-
-    # Summary
-    print("\n" + "=" * 70)
-    print("TEST SUMMARY")
-    print("=" * 70)
-
-    all_passed = True
-    for test_name, passed in results:
-        status = "✓ PASS" if passed else "✗ FAIL"
-        print(f"{status}: {test_name}")
-        all_passed = all_passed and passed
-
-    print("=" * 70)
-
-    if all_passed:
-        print("\n✓✓✓ ALL TESTS PASSED ✓✓✓")
-        print("✓ Security vulnerability is resolved")
-        print("✓ pyOpenSSL >= 25.3.0 is working correctly")
-        print("✓ cryptography >= 45.0.7 (not vulnerable)")
-        print("\nThe dependency update is safe to merge.\n")
-        return True
-    else:
-        print("\n✗✗✗ SOME TESTS FAILED ✗✗✗")
-        print("✗ Security requirements not met")
-        print("\nDo NOT merge until all tests pass.\n")
-        return False
-
-
-if __name__ == "__main__":
-    try:
-        success = main()
-        sys.exit(0 if success else 1)
-    except KeyboardInterrupt:
-        print("\n\nTest interrupted by user")
-        sys.exit(1)
-    except Exception as e:
-        print(f"\n✗ Unexpected error: {e}")
-        import traceback
-        traceback.print_exc()
-        sys.exit(1)

tests/test_pyopenssl_update.py

@@ -1,184 +0,0 @@
-"""
-Test script to verify pyOpenSSL update doesn't break crawl4ai functionality.
-
-This test verifies:
-1. pyOpenSSL and cryptography versions are correct and secure
-2. Basic crawling functionality still works
-3. HTTPS/SSL connections work properly
-4. Stealth mode integration works (uses playwright-stealth internally)
-
-Issue: #1545 - Security vulnerability in cryptography package
-Fix: Updated pyOpenSSL from >=24.3.0 to >=25.3.0
-Expected: cryptography package should be >=45.0.7 (above vulnerable range)
-"""
-
-import asyncio
-import sys
-from packaging import version
-
-
-def check_versions():
-    """Verify pyOpenSSL and cryptography versions meet security requirements."""
-    print("=" * 60)
-    print("STEP 1: Checking Package Versions")
-    print("=" * 60)
-
-    try:
-        import OpenSSL
-        pyopenssl_version = OpenSSL.__version__
-        print(f"✓ pyOpenSSL version: {pyopenssl_version}")
-
-        # Check pyOpenSSL >= 25.3.0
-        if version.parse(pyopenssl_version) >= version.parse("25.3.0"):
-            print(f"  ✓ Version check passed: {pyopenssl_version} >= 25.3.0")
-        else:
-            print(f"  ✗ Version check FAILED: {pyopenssl_version} < 25.3.0")
-            return False
-
-    except ImportError as e:
-        print(f"✗ Failed to import pyOpenSSL: {e}")
-        return False
-
-    try:
-        import cryptography
-        crypto_version = cryptography.__version__
-        print(f"✓ cryptography version: {crypto_version}")
-
-        # Check cryptography >= 45.0.7 (above vulnerable range)
-        if version.parse(crypto_version) >= version.parse("45.0.7"):
-            print(f"  ✓ Security check passed: {crypto_version} >= 45.0.7 (not vulnerable)")
-        else:
-            print(f"  ✗ Security check FAILED: {crypto_version} < 45.0.7 (potentially vulnerable)")
-            return False
-
-    except ImportError as e:
-        print(f"✗ Failed to import cryptography: {e}")
-        return False
-
-    print("\n✓ All version checks passed!\n")
-    return True
-
-
-async def test_basic_crawl():
-    """Test basic crawling functionality with HTTPS site."""
-    print("=" * 60)
-    print("STEP 2: Testing Basic HTTPS Crawling")
-    print("=" * 60)
-
-    try:
-        from crawl4ai import AsyncWebCrawler
-
-        async with AsyncWebCrawler(verbose=True) as crawler:
-            # Test with a simple HTTPS site (requires SSL/TLS)
-            print("Crawling example.com (HTTPS)...")
-            result = await crawler.arun(
-                url="https://www.example.com",
-                bypass_cache=True
-            )
-
-            if result.success:
-                print(f"✓ Crawl successful!")
-                print(f"  - Status code: {result.status_code}")
-                print(f"  - Content length: {len(result.html)} bytes")
-                print(f"  - SSL/TLS connection: ✓ Working")
-                return True
-            else:
-                print(f"✗ Crawl failed: {result.error_message}")
-                return False
-
-    except Exception as e:
-        print(f"✗ Test failed with error: {e}")
-        import traceback
-        traceback.print_exc()
-        return False
-
-
-async def test_stealth_mode():
-    """Test stealth mode functionality (depends on playwright-stealth)."""
-    print("\n" + "=" * 60)
-    print("STEP 3: Testing Stealth Mode Integration")
-    print("=" * 60)
-
-    try:
-        from crawl4ai import AsyncWebCrawler, BrowserConfig
-
-        # Create browser config with stealth mode
-        browser_config = BrowserConfig(
-            headless=True,
-            verbose=False
-        )
-
-        async with AsyncWebCrawler(config=browser_config, verbose=True) as crawler:
-            print("Crawling with stealth mode enabled...")
-            result = await crawler.arun(
-                url="https://www.example.com",
-                bypass_cache=True
-            )
-
-            if result.success:
-                print(f"✓ Stealth crawl successful!")
-                print(f"  - Stealth mode: ✓ Working")
-                return True
-            else:
-                print(f"✗ Stealth crawl failed: {result.error_message}")
-                return False
-
-    except Exception as e:
-        print(f"✗ Stealth test failed with error: {e}")
-        import traceback
-        traceback.print_exc()
-        return False
-
-
-async def main():
-    """Run all tests."""
-    print("\n")
-    print("╔" + "=" * 58 + "╗")
-    print("║  pyOpenSSL Security Update Verification Test (Issue #1545) ║")
-    print("╚" + "=" * 58 + "╝")
-    print("\n")
-
-    # Step 1: Check versions
-    versions_ok = check_versions()
-    if not versions_ok:
-        print("\n✗ FAILED: Version requirements not met")
-        return False
-
-    # Step 2: Test basic crawling
-    crawl_ok = await test_basic_crawl()
-    if not crawl_ok:
-        print("\n✗ FAILED: Basic crawling test failed")
-        return False
-
-    # Step 3: Test stealth mode
-    stealth_ok = await test_stealth_mode()
-    if not stealth_ok:
-        print("\n✗ FAILED: Stealth mode test failed")
-        return False
-
-    # All tests passed
-    print("\n" + "=" * 60)
-    print("FINAL RESULT")
-    print("=" * 60)
-    print("✓ All tests passed successfully!")
-    print("✓ pyOpenSSL update is working correctly")
-    print("✓ No breaking changes detected")
-    print("✓ Security vulnerability resolved")
-    print("=" * 60)
-    print("\n")
-
-    return True
-
-
-if __name__ == "__main__":
-    try:
-        success = asyncio.run(main())
-        sys.exit(0 if success else 1)
-    except KeyboardInterrupt:
-        print("\n\nTest interrupted by user")
-        sys.exit(1)
-    except Exception as e:
-        print(f"\n✗ Unexpected error: {e}")
-        import traceback
-        traceback.print_exc()
-        sys.exit(1)