fix: #855

feat(deep-crawling): enhance dispatcher to handle multi-page crawl results

Modify MemoryAdaptiveDispatcher to properly handle results from deep crawling operations:
- Add support for processing multiple results from deep crawling
- Implement memory usage distribution across multiple results
- Update task monitoring for deep crawling scenarios
- Modify return types in deep crawling strategies to use RunManyReturn

crawl4ai/__init__.py

@@ -4,12 +4,6 @@ import warnings
 from .async_webcrawler import AsyncWebCrawler, CacheMode
 from .async_configs import BrowserConfig, CrawlerRunConfig, HTTPCrawlerConfig, LLMConfig
 
-from .pipeline.pipeline import (
-    Pipeline,
-    create_pipeline,
-)
-from .pipeline.crawler import Crawler
-
 from .content_scraping_strategy import (
     ContentScrapingStrategy,
     WebScrapingStrategy,
@@ -71,14 +65,7 @@ from .deep_crawling import (
     DeepCrawlDecorator,
 )
 
-from .async_crawler_strategy import AsyncPlaywrightCrawlerStrategy, AsyncHTTPCrawlerStrategy
-
 __all__ = [
-    "Pipeline",
-    "AsyncPlaywrightCrawlerStrategy",
-    "AsyncHTTPCrawlerStrategy",
-    "create_pipeline",
-    "Crawler",
     "AsyncLoggerBase",
     "AsyncLogger",
     "AsyncWebCrawler",

crawl4ai/__version__.py

@@ -1,2 +1,2 @@
 # crawl4ai/_version.py
-__version__ = "0.5.0.post8"
+__version__ = "0.5.0.post4"

crawl4ai/async_configs.py

@@ -15,7 +15,7 @@ from .user_agent_generator import UAGen, ValidUAGenerator  # , OnlineUAGenerator
 from .extraction_strategy import ExtractionStrategy, LLMExtractionStrategy
 from .chunking_strategy import ChunkingStrategy, RegexChunking
 
-from .markdown_generation_strategy import MarkdownGenerationStrategy, DefaultMarkdownGenerator
+from .markdown_generation_strategy import MarkdownGenerationStrategy
 from .content_scraping_strategy import ContentScrapingStrategy, WebScrapingStrategy
 from .deep_crawling import DeepCrawlStrategy
 
@@ -29,7 +29,7 @@ from enum import Enum
 
 from .proxy_strategy import ProxyConfig
 try:
-    from .browser.models import DockerConfig
+    from .browser.docker_config import DockerConfig
 except ImportError:
     DockerConfig = None
 
@@ -176,7 +176,7 @@ class BrowserConfig:
         browser_mode (str): Determines how the browser should be initialized:
                            "builtin" - use the builtin CDP browser running in background
                            "dedicated" - create a new dedicated browser instance each time
-                           "cdp" - use explicit CDP settings provided in cdp_url
+                           "custom" - use explicit CDP settings provided in cdp_url
                            "docker" - run browser in Docker container with isolation
                            Default: "dedicated"
         use_managed_browser (bool): Launch the browser using a managed approach (e.g., via CDP), allowing
@@ -242,7 +242,7 @@ class BrowserConfig:
         channel: str = "chromium",
         proxy: str = None,
         proxy_config: Union[ProxyConfig, dict, None] = None,
-        docker_config: Union[DockerConfig, dict, None] = None,
+        docker_config: Union["DockerConfig", dict, None] = None,
         viewport_width: int = 1080,
         viewport_height: int = 600,
         viewport: dict = None,
@@ -270,7 +270,7 @@ class BrowserConfig:
         host: str = "localhost",
     ):
         self.browser_type = browser_type
-        self.headless = headless 
+        self.headless = headless
         self.browser_mode = browser_mode
         self.use_managed_browser = use_managed_browser
         self.cdp_url = cdp_url
@@ -289,10 +289,6 @@ class BrowserConfig:
             self.docker_config = DockerConfig.from_kwargs(docker_config)
         else:
             self.docker_config = docker_config
-
-        if self.docker_config:
-            self.user_data_dir = self.docker_config.user_data_dir
-
         self.viewport_width = viewport_width
         self.viewport_height = viewport_height
         self.viewport = viewport
@@ -722,7 +718,7 @@ class CrawlerRunConfig():
         word_count_threshold: int = MIN_WORD_THRESHOLD,
         extraction_strategy: ExtractionStrategy = None,
         chunking_strategy: ChunkingStrategy = RegexChunking(),
-        markdown_generator: MarkdownGenerationStrategy = DefaultMarkdownGenerator(),
+        markdown_generator: MarkdownGenerationStrategy = None,
         only_text: bool = False,
         css_selector: str = None,
         target_elements: List[str] = None,

crawl4ai/async_crawler_strategy.py

@@ -505,10 +505,7 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
             )
 
         # Get page for session
-        try:
-            page, context, _ = await self.browser_manager.get_page(crawlerRunConfig=config)
-        except Exception as e:
-            page, context = await self.browser_manager.get_page(crawlerRunConfig=config)
+        page, context = await self.browser_manager.get_page(crawlerRunConfig=config)
 
         # await page.goto(URL)
 
@@ -625,7 +622,7 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
             except Error:
                 visibility_info = await self.check_visibility(page)
 
-                if config.verbose:
+                if self.config.verbose:
                     self.logger.debug(
                         message="Body visibility info: {info}",
                         tag="DEBUG",

crawl4ai/async_dispatcher.py

@@ -1,4 +1,4 @@
-from typing import Dict, Optional, List, Tuple
+from typing import Dict, Optional, List, Tuple, Union
 from .async_configs import CrawlerRunConfig
 from .models import (
     CrawlResult,
@@ -183,7 +183,7 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
         config: CrawlerRunConfig,
         task_id: str,
         retry_count: int = 0,
-    ) -> CrawlerTaskResult:
+    ) -> Union[CrawlerTaskResult, List[CrawlerTaskResult]]:
         start_time = time.time()
         error_message = ""
         memory_usage = peak_memory = 0.0
@@ -244,8 +244,53 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
             end_memory = process.memory_info().rss / (1024 * 1024)
             memory_usage = peak_memory = end_memory - start_memory
             
-            # Handle rate limiting
-            if self.rate_limiter and result.status_code:
+            # Check if we have a container with multiple results (deep crawl result)
+            if isinstance(result, list) or (hasattr(result, '_results') and len(result._results) > 1):
+                # Handle deep crawling results - create a list of task results
+                task_results = []
+                result_list = result if isinstance(result, list) else result._results
+                
+                for idx, single_result in enumerate(result_list):
+                    # Create individual task result for each crawled page
+                    sub_task_id = f"{task_id}_{idx}"
+                    single_memory = memory_usage / len(result_list)  # Distribute memory usage
+                    
+                    # Only update rate limiter for first result which corresponds to the original URL
+                    if idx == 0 and self.rate_limiter and hasattr(single_result, 'status_code') and single_result.status_code:
+                        if not self.rate_limiter.update_delay(url, single_result.status_code):
+                            error_msg = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}"
+                            if self.monitor:
+                                self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+                    
+                    task_result = CrawlerTaskResult(
+                        task_id=sub_task_id,
+                        url=single_result.url,
+                        result=single_result,
+                        memory_usage=single_memory,
+                        peak_memory=single_memory,
+                        start_time=start_time,
+                        end_time=time.time(),
+                        error_message=single_result.error_message if not single_result.success else "",
+                        retry_count=retry_count
+                    )
+                    task_results.append(task_result)
+                
+                # Update monitor with completion status based on the first/primary result
+                if self.monitor:
+                    primary_result = result_list[0]
+                    if not primary_result.success:
+                        self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+                    else:
+                        self.monitor.update_task(
+                            task_id, 
+                            status=CrawlStatus.COMPLETED,
+                            extra_info=f"Deep crawl: {len(result_list)} pages"
+                        )
+                
+                return task_results
+            
+            # Handle single result (original behavior)
+            if self.rate_limiter and hasattr(result, 'status_code') and result.status_code:
                 if not self.rate_limiter.update_delay(url, result.status_code):
                     error_message = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}"
                     if self.monitor:
@@ -291,7 +336,7 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
             error_message=error_message,
             retry_count=retry_count
         )
-        
+            
     async def run_urls(
         self,
         urls: List[str],
@@ -356,8 +401,13 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
                     
                     # Process completed tasks
                     for completed_task in done:
-                        result = await completed_task
-                        results.append(result)
+                        task_result = await completed_task
+                        
+                        # Handle both single results and lists of results
+                        if isinstance(task_result, list):
+                            results.extend(task_result)
+                        else:
+                            results.append(task_result)
                         
                     # Update active tasks list
                     active_tasks = list(pending)
@@ -379,7 +429,7 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
             memory_monitor.cancel()
             if self.monitor:
                 self.monitor.stop()
-                
+
     async def _update_queue_priorities(self):
         """Periodically update priorities of items in the queue to prevent starvation"""
         # Skip if queue is empty

crawl4ai/async_logger.py

@@ -156,22 +156,9 @@ class AsyncLogger(AsyncLoggerBase):
                 formatted_message = message.format(**params)
 
                 # Then apply colors if specified
-                color_map = {
-                    "green": Fore.GREEN,
-                    "red": Fore.RED,
-                    "yellow": Fore.YELLOW,
-                    "blue": Fore.BLUE,
-                    "cyan": Fore.CYAN,
-                    "magenta": Fore.MAGENTA,
-                    "white": Fore.WHITE,
-                    "black": Fore.BLACK,
-                    "reset": Style.RESET_ALL,
-                }
                 if colors:
                     for key, color in colors.items():
                         # Find the formatted value in the message and wrap it with color
-                        if color in color_map:
-                            color = color_map[color]
                         if key in params:
                             value_str = str(params[key])
                             formatted_message = formatted_message.replace(

crawl4ai/async_webcrawler.py

@@ -4,25 +4,18 @@ import sys
 import time
 from colorama import Fore
 from pathlib import Path
-from typing import Optional, List
+from typing import Optional, List, Generic, TypeVar
 import json
 import asyncio
 
 # from contextlib import nullcontext, asynccontextmanager
 from contextlib import asynccontextmanager
-from .models import (
-    CrawlResult,
-    MarkdownGenerationResult,
-    DispatchResult,
-    ScrapingResult,
-    CrawlResultContainer,
-    RunManyReturn
-)
+from .models import CrawlResult, MarkdownGenerationResult, DispatchResult, ScrapingResult
 from .async_database import async_db_manager
 from .chunking_strategy import *  # noqa: F403
 from .chunking_strategy import IdentityChunking
 from .content_filter_strategy import *  # noqa: F403
-from .extraction_strategy import *  # noqa: F403
+from .extraction_strategy import * # noqa: F403
 from .extraction_strategy import NoExtractionStrategy
 from .async_crawler_strategy import (
     AsyncCrawlerStrategy,
@@ -37,7 +30,7 @@ from .markdown_generation_strategy import (
 from .deep_crawling import DeepCrawlDecorator
 from .async_logger import AsyncLogger, AsyncLoggerBase
 from .async_configs import BrowserConfig, CrawlerRunConfig
-from .async_dispatcher import *  # noqa: F403
+from .async_dispatcher import * # noqa: F403
 from .async_dispatcher import BaseDispatcher, MemoryAdaptiveDispatcher, RateLimiter
 
 from .utils import (
@@ -49,6 +42,45 @@ from .utils import (
     RobotsParser,
 )
 
+from typing import Union, AsyncGenerator
+
+CrawlResultT = TypeVar('CrawlResultT', bound=CrawlResult)
+# RunManyReturn = Union[CrawlResultT, List[CrawlResultT], AsyncGenerator[CrawlResultT, None]]
+
+class CrawlResultContainer(Generic[CrawlResultT]):
+    def __init__(self, results: Union[CrawlResultT, List[CrawlResultT]]):
+        # Normalize to a list
+        if isinstance(results, list):
+            self._results = results
+        else:
+            self._results = [results]
+
+    def __iter__(self):
+        return iter(self._results)
+
+    def __getitem__(self, index):
+        return self._results[index]
+
+    def __len__(self):
+        return len(self._results)
+
+    def __getattr__(self, attr):
+        # Delegate attribute access to the first element.
+        if self._results:
+            return getattr(self._results[0], attr)
+        raise AttributeError(f"{self.__class__.__name__} object has no attribute '{attr}'")
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}({self._results!r})"
+
+# Redefine the union type. Now synchronous calls always return a container,
+# while stream mode is handled with an AsyncGenerator.
+RunManyReturn = Union[
+    CrawlResultContainer[CrawlResultT],
+    AsyncGenerator[CrawlResultT, None]
+]
+
+
 
 class AsyncWebCrawler:
     """
@@ -161,18 +193,45 @@ class AsyncWebCrawler:
 
         # Decorate arun method with deep crawling capabilities
         self._deep_handler = DeepCrawlDecorator(self)
-        self.arun = self._deep_handler(self.arun)
+        self.arun = self._deep_handler(self.arun)  
 
     async def start(self):
         """
         Start the crawler explicitly without using context manager.
         This is equivalent to using 'async with' but gives more control over the lifecycle.
+
+        This method will:
+        1. Check for builtin browser if browser_mode is 'builtin'
+        2. Initialize the browser and context
+        3. Perform warmup sequence
+        4. Return the crawler instance for method chaining
+
         Returns:
             AsyncWebCrawler: The initialized crawler instance
         """
+        # Check for builtin browser if requested
+        if self.browser_config.browser_mode == "builtin" and not self.browser_config.cdp_url:
+            # Import here to avoid circular imports
+            from .browser_profiler import BrowserProfiler
+            profiler = BrowserProfiler(logger=self.logger)
+            
+            # Get builtin browser info or launch if needed
+            browser_info = profiler.get_builtin_browser_info()
+            if not browser_info:
+                self.logger.info("Builtin browser not found, launching new instance...", tag="BROWSER")
+                cdp_url = await profiler.launch_builtin_browser()
+                if not cdp_url:
+                    self.logger.warning("Failed to launch builtin browser, falling back to dedicated browser", tag="BROWSER")
+                else:
+                    self.browser_config.cdp_url = cdp_url
+                    self.browser_config.use_managed_browser = True
+            else:
+                self.logger.info(f"Using existing builtin browser at {browser_info.get('cdp_url')}", tag="BROWSER")
+                self.browser_config.cdp_url = browser_info.get('cdp_url')
+                self.browser_config.use_managed_browser = True
+                
         await self.crawler_strategy.__aenter__()
-        self.logger.info(f"Crawl4AI {crawl4ai_version}", tag="INIT")
-        self.ready = True
+        await self.awarmup()
         return self
 
     async def close(self):
@@ -192,6 +251,18 @@ class AsyncWebCrawler:
     async def __aexit__(self, exc_type, exc_val, exc_tb):
         await self.close()
 
+    async def awarmup(self):
+        """
+        Initialize the crawler with warm-up sequence.
+
+        This method:
+        1. Logs initialization info
+        2. Sets up browser configuration
+        3. Marks the crawler as ready
+        """
+        self.logger.info(f"Crawl4AI {crawl4ai_version}", tag="INIT")
+        self.ready = True
+
     @asynccontextmanager
     async def nullcontext(self):
         """异步空上下文管理器"""
@@ -234,7 +305,7 @@ class AsyncWebCrawler:
         # Auto-start if not ready
         if not self.ready:
             await self.start()
-
+            
         config = config or CrawlerRunConfig()
         if not isinstance(url, str) or not url:
             raise ValueError("Invalid URL, make sure the URL is a non-empty string")
@@ -248,7 +319,9 @@ class AsyncWebCrawler:
                     config.cache_mode = CacheMode.ENABLED
 
                 # Create cache context
-                cache_context = CacheContext(url, config.cache_mode, False)
+                cache_context = CacheContext(
+                    url, config.cache_mode, False
+                )
 
                 # Initialize processing variables
                 async_response: AsyncCrawlResponse = None
@@ -278,7 +351,7 @@ class AsyncWebCrawler:
                     # if config.screenshot and not screenshot or config.pdf and not pdf:
                     if config.screenshot and not screenshot_data:
                         cached_result = None
-
+                    
                     if config.pdf and not pdf_data:
                         cached_result = None
 
@@ -310,18 +383,14 @@ class AsyncWebCrawler:
 
                     # Check robots.txt if enabled
                     if config and config.check_robots_txt:
-                        if not await self.robots_parser.can_fetch(
-                            url, self.browser_config.user_agent
-                        ):
+                        if not await self.robots_parser.can_fetch(url, self.browser_config.user_agent):
                             return CrawlResult(
                                 url=url,
                                 html="",
                                 success=False,
                                 status_code=403,
                                 error_message="Access denied by robots.txt",
-                                response_headers={
-                                    "X-Robots-Status": "Blocked by robots.txt"
-                                },
+                                response_headers={"X-Robots-Status": "Blocked by robots.txt"}
                             )
 
                     ##############################
@@ -348,7 +417,7 @@ class AsyncWebCrawler:
                     ###############################################################
                     # Process the HTML content, Call CrawlerStrategy.process_html #
                     ###############################################################
-                    crawl_result: CrawlResult = await self.aprocess_html(
+                    crawl_result : CrawlResult = await self.aprocess_html(
                         url=url,
                         html=html,
                         extracted_content=extracted_content,
@@ -425,7 +494,7 @@ class AsyncWebCrawler:
                     tag="ERROR",
                 )
 
-                return CrawlResultContainer(
+                return  CrawlResultContainer(
                     CrawlResult(
                         url=url, html="", success=False, error_message=error_message
                     )
@@ -470,14 +539,15 @@ class AsyncWebCrawler:
 
             # Process HTML content
             params = config.__dict__.copy()
-            params.pop("url", None)
+            params.pop("url", None)            
             # add keys from kwargs to params that doesn't exist in params
             params.update({k: v for k, v in kwargs.items() if k not in params.keys()})
 
+            
             ################################
             # Scraping Strategy Execution  #
             ################################
-            result: ScrapingResult = scraping_strategy.scrap(url, html, **params)
+            result : ScrapingResult = scraping_strategy.scrap(url, html, **params)
 
             if result is None:
                 raise ValueError(
@@ -526,10 +596,7 @@ class AsyncWebCrawler:
         self.logger.info(
             message="{url:.50}... | Time: {timing}s",
             tag="SCRAPE",
-            params={
-                "url": _url,
-                "timing": int((time.perf_counter() - t1) * 1000) / 1000,
-            },
+            params={"url": _url, "timing": int((time.perf_counter() - t1) * 1000) / 1000},
         )
 
         ################################
@@ -604,22 +671,10 @@ class AsyncWebCrawler:
     async def arun_many(
         self,
         urls: List[str],
-        config: Optional[CrawlerRunConfig] = None,
+        config: Optional[CrawlerRunConfig] = None, 
         dispatcher: Optional[BaseDispatcher] = None,
-        # Legacy parameters maintained for backwards compatibility
-        # word_count_threshold=MIN_WORD_THRESHOLD,
-        # extraction_strategy: ExtractionStrategy = None,
-        # chunking_strategy: ChunkingStrategy = RegexChunking(),
-        # content_filter: RelevantContentFilter = None,
-        # cache_mode: Optional[CacheMode] = None,
-        # bypass_cache: bool = False,
-        # css_selector: str = None,
-        # screenshot: bool = False,
-        # pdf: bool = False,
-        # user_agent: str = None,
-        # verbose=True,
-        **kwargs,
-    ) -> RunManyReturn:
+        **kwargs
+        ) -> RunManyReturn:
         """
         Runs the crawler for multiple URLs concurrently using a configurable dispatcher strategy.
 
@@ -651,20 +706,7 @@ class AsyncWebCrawler:
             print(f"Processed {result.url}: {len(result.markdown)} chars")
         """
         config = config or CrawlerRunConfig()
-        # if config is None:
-        #     config = CrawlerRunConfig(
-        #         word_count_threshold=word_count_threshold,
-        #         extraction_strategy=extraction_strategy,
-        #         chunking_strategy=chunking_strategy,
-        #         content_filter=content_filter,
-        #         cache_mode=cache_mode,
-        #         bypass_cache=bypass_cache,
-        #         css_selector=css_selector,
-        #         screenshot=screenshot,
-        #         pdf=pdf,
-        #         verbose=verbose,
-        #         **kwargs,
-        #     )
+
 
         if dispatcher is None:
             dispatcher = MemoryAdaptiveDispatcher(
@@ -675,32 +717,37 @@ class AsyncWebCrawler:
 
         def transform_result(task_result):
             return (
-                setattr(
-                    task_result.result,
-                    "dispatch_result",
-                    DispatchResult(
-                        task_id=task_result.task_id,
-                        memory_usage=task_result.memory_usage,
-                        peak_memory=task_result.peak_memory,
-                        start_time=task_result.start_time,
-                        end_time=task_result.end_time,
-                        error_message=task_result.error_message,
-                    ),
+                    setattr(task_result.result, 'dispatch_result', 
+                        DispatchResult(
+                            task_id=task_result.task_id,
+                            memory_usage=task_result.memory_usage,
+                            peak_memory=task_result.peak_memory,
+                            start_time=task_result.start_time,
+                            end_time=task_result.end_time,
+                            error_message=task_result.error_message,
+                        )
+                    ) or task_result.result
                 )
-                or task_result.result
-            )
 
         stream = config.stream
-
+        
         if stream:
-
             async def result_transformer():
-                async for task_result in dispatcher.run_urls_stream(
-                    crawler=self, urls=urls, config=config
-                ):
+                async for task_result in dispatcher.run_urls_stream(crawler=self, urls=urls, config=config):
                     yield transform_result(task_result)
-
             return result_transformer()
         else:
             _results = await dispatcher.run_urls(crawler=self, urls=urls, config=config)
-            return [transform_result(res) for res in _results]
+            return [transform_result(res) for res in _results]    
+
+    async def aclear_cache(self):
+        """Clear the cache database."""
+        await async_db_manager.cleanup()
+
+    async def aflush_cache(self):
+        """Flush the cache database."""
+        await async_db_manager.aflush_db()
+
+    async def aget_cache_size(self):
+        """Get the total number of cached items."""
+        return await async_db_manager.aget_total_count()

crawl4ai/browser/__init__.py

@@ -6,18 +6,5 @@ for browser creation and interaction.
 
 from .manager import BrowserManager
 from .profiles import BrowserProfileManager
-from .models import DockerConfig
-from .docker_registry import DockerRegistry
-from .docker_utils import DockerUtils
-from .browser_hub import BrowserHub
-from .strategies import (
-    BaseBrowserStrategy,
-    PlaywrightBrowserStrategy,
-    CDPBrowserStrategy,
-    BuiltinBrowserStrategy,
-    DockerBrowserStrategy
-)
 
-__all__ = ['BrowserManager', 'BrowserProfileManager', 'DockerConfig', 'DockerRegistry', 'DockerUtils', 'BaseBrowserStrategy',
-           'PlaywrightBrowserStrategy', 'CDPBrowserStrategy', 'BuiltinBrowserStrategy',
-           'DockerBrowserStrategy', 'BrowserHub']
+__all__ = ['BrowserManager', 'BrowserProfileManager']

crawl4ai/browser/browser_hub.py

@@ -1,184 +0,0 @@
-# browser_hub_manager.py
-import hashlib
-import json
-import asyncio
-from typing import Dict, Optional, List, Tuple
-from .manager import BrowserManager, UnavailableBehavior
-from ..async_configs import BrowserConfig, CrawlerRunConfig
-from ..async_logger import AsyncLogger
-
-class BrowserHub:
-    """
-    Manages Browser-Hub instances for sharing across multiple pipelines.
-    
-    This class provides centralized management for browser resources, allowing
-    multiple pipelines to share browser instances efficiently, connect to
-    existing browser hubs, or create new ones with custom configurations.
-    """
-    _instances: Dict[str, BrowserManager] = {}
-    _lock = asyncio.Lock()
-    
-    @classmethod
-    async def get_browser_manager(
-        cls, 
-        config: Optional[BrowserConfig] = None,
-        hub_id: Optional[str] = None,
-        connection_info: Optional[str] = None,
-        logger: Optional[AsyncLogger] = None,
-        max_browsers_per_config: int = 10,
-        max_pages_per_browser: int = 5,
-        initial_pool_size: int = 1,
-        page_configs: Optional[List[Tuple[BrowserConfig, CrawlerRunConfig, int]]] = None
-    ) -> BrowserManager:
-        """
-        Get an existing BrowserManager or create a new one based on parameters.
-        
-        Args:
-            config: Browser configuration for new hub
-            hub_id: Identifier for the hub instance
-            connection_info: Connection string for existing hub
-            logger: Logger for recording events and errors
-            max_browsers_per_config: Maximum browsers per configuration
-            max_pages_per_browser: Maximum pages per browser
-            initial_pool_size: Initial number of browsers to create
-            page_configs: Optional configurations for pre-warming pages
-            
-        Returns:
-            BrowserManager: The requested browser manager instance
-        """
-        async with cls._lock:
-            # Scenario 3: Use existing hub via connection info
-            if connection_info:
-                instance_key = f"connection:{connection_info}"
-                if instance_key not in cls._instances:
-                    cls._instances[instance_key] = await cls._connect_to_browser_hub(
-                        connection_info, logger
-                    )
-                return cls._instances[instance_key]
-                
-            # Scenario 2: Custom configured hub
-            if config:
-                config_hash = cls._hash_config(config)
-                instance_key = hub_id or f"config:{config_hash}"
-                if instance_key not in cls._instances:
-                    cls._instances[instance_key] = await cls._create_browser_manager(
-                        config, 
-                        logger,
-                        max_browsers_per_config,
-                        max_pages_per_browser,
-                        initial_pool_size,
-                        page_configs
-                    )
-                return cls._instances[instance_key]
-            
-            # Scenario 1: Default hub
-            instance_key = "default"
-            if instance_key not in cls._instances:
-                cls._instances[instance_key] = await cls._create_default_browser_hub(
-                    logger,
-                    max_browsers_per_config,
-                    max_pages_per_browser,
-                    initial_pool_size
-                )
-            return cls._instances[instance_key]
-    
-    @classmethod
-    async def _create_browser_manager(
-        cls, 
-        config: BrowserConfig,
-        logger: Optional[AsyncLogger],
-        max_browsers_per_config: int,
-        max_pages_per_browser: int,
-        initial_pool_size: int,
-        page_configs: Optional[List[Tuple[BrowserConfig, CrawlerRunConfig, int]]] = None
-    ) -> BrowserManager:
-        """Create a new browser hub with the specified configuration."""
-        manager = BrowserManager(
-            browser_config=config,
-            logger=logger,
-            unavailable_behavior=UnavailableBehavior.ON_DEMAND,
-            max_browsers_per_config=max_browsers_per_config,
-            max_pages_per_browser=max_pages_per_browser,
-        )
-        
-        # Initialize the pool
-        await manager.initialize_pool(
-            browser_configs=[config] if config else None,
-            browsers_per_config=initial_pool_size,
-            page_configs=page_configs
-        )
-        
-        return manager
-    
-    @classmethod
-    async def _create_default_browser_hub(
-        cls,
-        logger: Optional[AsyncLogger],
-        max_browsers_per_config: int,
-        max_pages_per_browser: int,
-        initial_pool_size: int
-    ) -> BrowserManager:
-        """Create a default browser hub with standard settings."""
-        config = BrowserConfig(headless=True)
-        return await cls._create_browser_manager(
-            config, 
-            logger, 
-            max_browsers_per_config, 
-            max_pages_per_browser, 
-            initial_pool_size,
-            None
-        )
-    
-    @classmethod
-    async def _connect_to_browser_hub(
-        cls, 
-        connection_info: str,
-        logger: Optional[AsyncLogger]
-    ) -> BrowserManager:
-        """
-        Connect to an existing browser hub.
-        
-        Note: This is a placeholder for future remote connection functionality.
-        Currently creates a local instance.
-        """
-        if logger:
-            logger.info(
-                message="Remote browser hub connections not yet implemented. Creating local instance.",
-                tag="BROWSER_HUB"
-            )
-        # For now, create a default local instance
-        return await cls._create_default_browser_hub(
-            logger, 
-            max_browsers_per_config=10, 
-            max_pages_per_browser=5, 
-            initial_pool_size=1
-        )
-    
-    @classmethod
-    def _hash_config(cls, config: BrowserConfig) -> str:
-        """Create a hash of the browser configuration for identification."""
-        # Convert config to dictionary, excluding any callable objects
-        config_dict = config.__dict__.copy()
-        for key in list(config_dict.keys()):
-            if callable(config_dict[key]):
-                del config_dict[key]
-        
-        # Convert to canonical JSON string
-        config_json = json.dumps(config_dict, sort_keys=True, default=str)
-        
-        # Hash the JSON
-        config_hash = hashlib.sha256(config_json.encode()).hexdigest()
-        return config_hash
-    
-    @classmethod
-    async def shutdown_all(cls):
-        """Close all browser hub instances and clear the registry."""
-        async with cls._lock:
-            shutdown_tasks = []
-            for hub in cls._instances.values():
-                shutdown_tasks.append(hub.close())
-            
-            if shutdown_tasks:
-                await asyncio.gather(*shutdown_tasks)
-            
-            cls._instances.clear()

crawl4ai/browser/docker/alpine/connect.Dockerfile

@@ -1,34 +0,0 @@
-# ---------- Dockerfile ----------
-    FROM alpine:latest
-
-    # Combine everything in one RUN to keep layers minimal.
-    RUN apk update && apk upgrade && \
-        apk add --no-cache \
-            chromium \
-            nss \
-            freetype \
-            harfbuzz \
-            ca-certificates \
-            ttf-freefont \
-            socat \
-            curl && \
-        addgroup -S chromium && adduser -S chromium -G chromium && \
-        mkdir -p /data && chown chromium:chromium /data && \
-        rm -rf /var/cache/apk/*
-    
-    # Copy start script, then chown/chmod in one step
-    COPY start.sh /home/chromium/start.sh
-    RUN chown chromium:chromium /home/chromium/start.sh && \
-        chmod +x /home/chromium/start.sh
-    
-    USER chromium
-    WORKDIR /home/chromium
-    
-    # Expose port used by socat (mapping 9222→9223 or whichever you prefer)
-    EXPOSE 9223
-    
-    # Simple healthcheck: is the remote debug endpoint responding?
-    HEALTHCHECK --interval=30s --timeout=5s --retries=3 CMD curl -f http://localhost:9222/json/version || exit 1
-    
-    CMD ["./start.sh"]
-    

crawl4ai/browser/docker/alpine/launch.Dockerfile

@@ -1,27 +0,0 @@
-# ---------- Dockerfile (Idle Version) ----------
-    FROM alpine:latest
-
-    # Install only Chromium and its dependencies in a single layer
-    RUN apk update && apk upgrade && \
-        apk add --no-cache \
-            chromium \
-            nss \
-            freetype \
-            harfbuzz \
-            ca-certificates \
-            ttf-freefont \
-            socat \
-            curl && \
-        addgroup -S chromium && adduser -S chromium -G chromium && \
-        mkdir -p /data && chown chromium:chromium /data && \
-        rm -rf /var/cache/apk/*
-    
-    ENV PATH="/usr/bin:/bin:/usr/sbin:/sbin"
-
-    # Switch to a non-root user for security
-    USER chromium
-    WORKDIR /home/chromium
-    
-    # Idle: container does nothing except stay alive
-    CMD ["tail", "-f", "/dev/null"]
-    

crawl4ai/browser/docker/connect.Dockerfile

@@ -0,0 +1,61 @@
+FROM ubuntu:22.04
+
+# Install dependencies with comprehensive Chromium support
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    wget \
+    gnupg \
+    ca-certificates \
+    fonts-liberation \
+    # Sound support
+    libasound2 \
+    # Accessibility support
+    libatspi2.0-0 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    # Graphics and rendering
+    libdrm2 \
+    libgbm1 \
+    libgtk-3-0 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxext6 \
+    libxfixes3 \
+    libxrandr2 \
+    # X11 and window system
+    libx11-6 \
+    libxcb1 \
+    libxkbcommon0 \
+    # Text and internationalization
+    libpango-1.0-0 \
+    libcairo2 \
+    # Printing support
+    libcups2 \
+    # System libraries
+    libdbus-1-3 \
+    libnss3 \
+    libnspr4 \
+    libglib2.0-0 \
+    # Utilities
+    xdg-utils \
+    socat \
+    # Process management
+    procps \
+    # Clean up
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Chrome
+RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
+    echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \
+    apt-get update && \
+    apt-get install -y google-chrome-stable && \
+    rm -rf /var/lib/apt/lists/*
+
+# Create data directory for user data
+RUN mkdir -p /data && chmod 777 /data
+
+# Add a startup script
+COPY start.sh /start.sh
+RUN chmod +x /start.sh
+
+# Set entrypoint
+ENTRYPOINT ["/start.sh"]

crawl4ai/browser/docker/debian/connect.Dockerfile

@@ -1,23 +0,0 @@
-# Use Debian 12 (Bookworm) slim for a small, stable base image
-FROM debian:bookworm-slim
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-# Install Chromium, socat, and basic fonts
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    chromium \
-    wget \
-    curl \
-    socat \
-    fonts-freefont-ttf \
-    fonts-noto-color-emoji && \
-    apt-get clean && rm -rf /var/lib/apt/lists/*
-
-# Copy start.sh and make it executable
-COPY start.sh /start.sh
-RUN chmod +x /start.sh
-
-# Expose socat port (use host mapping, e.g. -p 9225:9223)
-EXPOSE 9223
-
-ENTRYPOINT ["/start.sh"]

crawl4ai/browser/docker/launch.Dockerfile

@@ -0,0 +1,57 @@
+FROM ubuntu:22.04
+
+# Install dependencies with comprehensive Chromium support
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    wget \
+    gnupg \
+    ca-certificates \
+    fonts-liberation \
+    # Sound support
+    libasound2 \
+    # Accessibility support
+    libatspi2.0-0 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    # Graphics and rendering
+    libdrm2 \
+    libgbm1 \
+    libgtk-3-0 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxext6 \
+    libxfixes3 \
+    libxrandr2 \
+    # X11 and window system
+    libx11-6 \
+    libxcb1 \
+    libxkbcommon0 \
+    # Text and internationalization
+    libpango-1.0-0 \
+    libcairo2 \
+    # Printing support
+    libcups2 \
+    # System libraries
+    libdbus-1-3 \
+    libnss3 \
+    libnspr4 \
+    libglib2.0-0 \
+    # Utilities
+    xdg-utils \
+    socat \
+    # Process management
+    procps \
+    # Clean up
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Chrome
+RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
+    echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \
+    apt-get update && \
+    apt-get install -y google-chrome-stable && \
+    rm -rf /var/lib/apt/lists/*
+
+# Create data directory for user data
+RUN mkdir -p /data && chmod 777 /data
+
+# Keep container running without starting Chrome
+CMD ["tail", "-f", "/dev/null"]

crawl4ai/browser/docker_config.py

@@ -0,0 +1,133 @@
+"""Docker configuration module for Crawl4AI browser automation.
+
+This module provides configuration classes for Docker-based browser automation,
+allowing flexible configuration of Docker containers for browsing.
+"""
+
+from typing import Dict, List, Optional, Union
+
+
+class DockerConfig:
+    """Configuration for Docker-based browser automation.
+    
+    This class contains Docker-specific settings to avoid cluttering BrowserConfig.
+    
+    Attributes:
+        mode (str): Docker operation mode - "connect" or "launch".
+            - "connect": Uses a container with Chrome already running
+            - "launch": Dynamically configures and starts Chrome in container
+        image (str): Docker image to use. If None, defaults from DockerUtils are used.
+        registry_file (str): Path to container registry file for persistence.
+        persistent (bool): Keep container running after browser closes.
+        remove_on_exit (bool): Remove container on exit when not persistent.
+        network (str): Docker network to use.
+        volumes (List[str]): Volume mappings (e.g., ["host_path:container_path"]).
+        env_vars (Dict[str, str]): Environment variables to set in container.
+        extra_args (List[str]): Additional docker run arguments.
+        host_port (int): Host port to map to container's 9223 port.
+        user_data_dir (str): Path to user data directory on host.
+        container_user_data_dir (str): Path to user data directory in container.
+    """
+    
+    def __init__(
+        self,
+        mode: str = "connect",                     # "connect" or "launch" 
+        image: Optional[str] = None,               # Docker image to use
+        registry_file: Optional[str] = None,       # Path to registry file
+        persistent: bool = False,                  # Keep container running after browser closes
+        remove_on_exit: bool = True,               # Remove container on exit when not persistent
+        network: Optional[str] = None,             # Docker network to use
+        volumes: List[str] = None,                 # Volume mappings
+        env_vars: Dict[str, str] = None,           # Environment variables
+        extra_args: List[str] = None,              # Additional docker run arguments
+        host_port: Optional[int] = None,           # Host port to map to container's 9223
+        user_data_dir: Optional[str] = None,       # Path to user data directory on host
+        container_user_data_dir: str = "/data",    # Path to user data directory in container
+    ):
+        """Initialize Docker configuration.
+        
+        Args:
+            mode: Docker operation mode ("connect" or "launch")
+            image: Docker image to use
+            registry_file: Path to container registry file
+            persistent: Whether to keep container running after browser closes
+            remove_on_exit: Whether to remove container on exit when not persistent
+            network: Docker network to use
+            volumes: Volume mappings as list of strings
+            env_vars: Environment variables as dictionary
+            extra_args: Additional docker run arguments
+            host_port: Host port to map to container's 9223
+            user_data_dir: Path to user data directory on host
+            container_user_data_dir: Path to user data directory in container
+        """
+        self.mode = mode
+        self.image = image  # If None, defaults will be used from DockerUtils
+        self.registry_file = registry_file
+        self.persistent = persistent
+        self.remove_on_exit = remove_on_exit
+        self.network = network
+        self.volumes = volumes or []
+        self.env_vars = env_vars or {}
+        self.extra_args = extra_args or []
+        self.host_port = host_port
+        self.user_data_dir = user_data_dir
+        self.container_user_data_dir = container_user_data_dir
+    
+    def to_dict(self) -> Dict:
+        """Convert this configuration to a dictionary.
+        
+        Returns:
+            Dictionary representation of this configuration
+        """
+        return {
+            "mode": self.mode,
+            "image": self.image,
+            "registry_file": self.registry_file,
+            "persistent": self.persistent,
+            "remove_on_exit": self.remove_on_exit,
+            "network": self.network,
+            "volumes": self.volumes,
+            "env_vars": self.env_vars,
+            "extra_args": self.extra_args,
+            "host_port": self.host_port,
+            "user_data_dir": self.user_data_dir,
+            "container_user_data_dir": self.container_user_data_dir
+        }
+        
+    @staticmethod
+    def from_kwargs(kwargs: Dict) -> "DockerConfig":
+        """Create a DockerConfig from a dictionary of keyword arguments.
+        
+        Args:
+            kwargs: Dictionary of configuration options
+            
+        Returns:
+            New DockerConfig instance
+        """
+        return DockerConfig(
+            mode=kwargs.get("mode", "connect"),
+            image=kwargs.get("image"),
+            registry_file=kwargs.get("registry_file"),
+            persistent=kwargs.get("persistent", False),
+            remove_on_exit=kwargs.get("remove_on_exit", True),
+            network=kwargs.get("network"),
+            volumes=kwargs.get("volumes"),
+            env_vars=kwargs.get("env_vars"),
+            extra_args=kwargs.get("extra_args"),
+            host_port=kwargs.get("host_port"),
+            user_data_dir=kwargs.get("user_data_dir"),
+            container_user_data_dir=kwargs.get("container_user_data_dir", "/data")
+        )
+        
+    def clone(self, **kwargs) -> "DockerConfig":
+        """Create a copy of this configuration with updated values.
+        
+        Args:
+            **kwargs: Key-value pairs of configuration options to update
+            
+        Returns:
+            DockerConfig: A new instance with the specified updates
+        """
+        config_dict = self.to_dict()
+        config_dict.update(kwargs)
+        return DockerConfig.from_kwargs(config_dict)

crawl4ai/browser/docker_registry.py

@@ -31,10 +31,9 @@ class DockerRegistry:
         Args:
             registry_file: Path to the registry file. If None, uses default path.
         """
-        # Use the same file path as BuiltinBrowserStrategy by default
-        self.registry_file = registry_file or os.path.join(get_home_folder(), "builtin-browser", "browser_config.json")
-        self.containers = {}  # Still maintain this for backward compatibility
-        self.port_map = {}    # Will be populated from the shared file
+        self.registry_file = registry_file or os.path.join(get_home_folder(), "docker_browser_registry.json")
+        self.containers = {}
+        self.port_map = {}
         self.last_port = 9222
         self.load()
     
@@ -44,35 +43,11 @@ class DockerRegistry:
             try:
                 with open(self.registry_file, 'r') as f:
                     registry_data = json.load(f)
-                    
-                    # Initialize port_map if not present
-                    if "port_map" not in registry_data:
-                        registry_data["port_map"] = {}
-                    
-                    self.port_map = registry_data.get("port_map", {})
-                    
-                    # Extract container information from port_map entries of type "docker"
-                    self.containers = {}
-                    for port_str, browser_info in self.port_map.items():
-                        if browser_info.get("browser_type") == "docker" and "container_id" in browser_info:
-                            container_id = browser_info["container_id"]
-                            self.containers[container_id] = {
-                                "host_port": int(port_str),
-                                "config_hash": browser_info.get("config_hash", ""),
-                                "created_at": browser_info.get("created_at", time.time())
-                            }
-                    
-                    # Get last port if available
-                    if "last_port" in registry_data:
-                        self.last_port = registry_data["last_port"]
-                    else:
-                        # Find highest port in port_map
-                        ports = [int(p) for p in self.port_map.keys() if p.isdigit()]
-                        self.last_port = max(ports + [9222])
-                    
-            except Exception as e:
+                    self.containers = registry_data.get("containers", {})
+                    self.port_map = registry_data.get("ports", {})
+                    self.last_port = registry_data.get("last_port", 9222)
+            except Exception:
                 # Reset to defaults on error
-                print(f"Error loading registry: {e}")
                 self.containers = {}
                 self.port_map = {}
                 self.last_port = 9222
@@ -84,75 +59,28 @@ class DockerRegistry:
     
     def save(self):
         """Save container registry to file."""
-        # First load the current file to avoid overwriting other browser types
-        current_data = {"port_map": {}, "last_port": self.last_port}
-        if os.path.exists(self.registry_file):
-            try:
-                with open(self.registry_file, 'r') as f:
-                    current_data = json.load(f)
-            except Exception:
-                pass
-        
-        # Create a new port_map dictionary
-        updated_port_map = {}
-        
-        # First, copy all non-docker entries from the existing port_map
-        for port_str, browser_info in current_data.get("port_map", {}).items():
-            if browser_info.get("browser_type") != "docker":
-                updated_port_map[port_str] = browser_info
-        
-        # Then add all current docker container entries
-        for container_id, container_info in self.containers.items():
-            port_str = str(container_info["host_port"])
-            updated_port_map[port_str] = {
-                "browser_type": "docker",
-                "container_id": container_id,
-                "cdp_url": f"http://localhost:{port_str}",
-                "config_hash": container_info["config_hash"],
-                "created_at": container_info["created_at"]
-            }
-        
-        # Replace the port_map with our updated version
-        current_data["port_map"] = updated_port_map
-        
-        # Update last_port
-        current_data["last_port"] = self.last_port
-        
-        # Ensure directory exists
         os.makedirs(os.path.dirname(self.registry_file), exist_ok=True)
-        
-        # Save the updated data
         with open(self.registry_file, 'w') as f:
-            json.dump(current_data, f, indent=2)
+            json.dump({
+                "containers": self.containers,
+                "ports": self.port_map,
+                "last_port": self.last_port
+            }, f, indent=2)
     
-    def register_container(self, container_id: str, host_port: int, config_hash: str, cdp_json_config: Optional[str] = None):
+    def register_container(self, container_id: str, host_port: int, config_hash: str):
         """Register a container with its configuration hash and port mapping.
         
         Args:
             container_id: Docker container ID
             host_port: Host port mapped to container
             config_hash: Hash of configuration used to create container
-            cdp_json_config: CDP JSON configuration if available
         """
         self.containers[container_id] = {
             "host_port": host_port,
             "config_hash": config_hash,
             "created_at": time.time()
         }
-        
-        # Update port_map to maintain compatibility with BuiltinBrowserStrategy
-        port_str = str(host_port)
-        self.port_map[port_str] = {
-            "browser_type": "docker",
-            "container_id": container_id,
-            "cdp_url": f"http://localhost:{port_str}",
-            "config_hash": config_hash,
-            "created_at": time.time()
-        }
-        
-        if cdp_json_config:
-            self.port_map[port_str]["cdp_json_config"] = cdp_json_config
-        
+        self.port_map[str(host_port)] = container_id
         self.save()
     
     def unregister_container(self, container_id: str):
@@ -163,18 +91,12 @@ class DockerRegistry:
         """
         if container_id in self.containers:
             host_port = self.containers[container_id]["host_port"]
-            port_str = str(host_port)
-            
-            # Remove from port_map
-            if port_str in self.port_map:
-                del self.port_map[port_str]
-                
-            # Remove from containers
+            if str(host_port) in self.port_map:
+                del self.port_map[str(host_port)]
             del self.containers[container_id]
-            
             self.save()
     
-    async def find_container_by_config(self, config_hash: str, docker_utils) -> Optional[str]:
+    def find_container_by_config(self, config_hash: str, docker_utils) -> Optional[str]:
         """Find a container that matches the given configuration hash.
         
         Args:
@@ -184,16 +106,9 @@ class DockerRegistry:
         Returns:
             Container ID if found, None otherwise
         """
-        # Search through port_map for entries with matching config_hash
-        for port_str, browser_info in self.port_map.items():
-            if (browser_info.get("browser_type") == "docker" and 
-                browser_info.get("config_hash") == config_hash and 
-                "container_id" in browser_info):
-                
-                container_id = browser_info["container_id"]
-                if await docker_utils.is_container_running(container_id):
-                    return container_id
-        
+        for container_id, data in self.containers.items():
+            if data["config_hash"] == config_hash and docker_utils.is_container_running(container_id):
+                return container_id
         return None
     
     def get_container_host_port(self, container_id: str) -> Optional[int]:
@@ -222,7 +137,7 @@ class DockerRegistry:
         port = self.last_port + 1
         
         # Check if port is in use (either in our registry or system-wide)
-        while str(port) in self.port_map or docker_utils.is_port_in_use(port):
+        while port in self.port_map or docker_utils.is_port_in_use(port):
             port += 1
         
         # Update last port
@@ -251,14 +166,9 @@ class DockerRegistry:
             docker_utils: DockerUtils instance to check container status
         """
         to_remove = []
-        
-        # Find containers that are no longer running
-        for port_str, browser_info in self.port_map.items():
-            if browser_info.get("browser_type") == "docker" and "container_id" in browser_info:
-                container_id = browser_info["container_id"]
-                if not docker_utils.is_container_running(container_id):
-                    to_remove.append(container_id)
-        
-        # Remove stale containers
+        for container_id in self.containers:
+            if not docker_utils.is_container_running(container_id):
+                to_remove.append(container_id)
+                
         for container_id in to_remove:
             self.unregister_container(container_id)

crawl4ai/browser/docker_strategy.py

@@ -0,0 +1,286 @@
+"""Docker browser strategy module for Crawl4AI.
+
+This module provides browser strategies for running browsers in Docker containers,
+which offers better isolation, consistency across platforms, and easy scaling.
+"""
+
+import os
+import uuid
+import asyncio
+from typing import Dict, List, Optional, Tuple, Union
+from pathlib import Path
+
+from playwright.async_api import Page, BrowserContext
+
+from ..async_logger import AsyncLogger
+from ..async_configs import BrowserConfig, CrawlerRunConfig
+from .docker_config import DockerConfig
+from .docker_registry import DockerRegistry
+from .docker_utils import DockerUtils
+from .strategies import BuiltinBrowserStrategy
+
+
+class DockerBrowserStrategy(BuiltinBrowserStrategy):
+    """Docker-based browser strategy.
+    
+    Extends the BuiltinBrowserStrategy to run browsers in Docker containers.
+    Supports two modes:
+    1. "connect" - Uses a Docker image with Chrome already running
+    2. "launch" - Starts Chrome within the container with custom settings
+    
+    Attributes:
+        docker_config: Docker-specific configuration options
+        container_id: ID of current Docker container
+        container_name: Name assigned to the container
+        registry: Registry for tracking and reusing containers
+        docker_utils: Utilities for Docker operations
+        chrome_process_id: Process ID of Chrome within container
+        socat_process_id: Process ID of socat within container
+        internal_cdp_port: Chrome's internal CDP port
+        internal_mapped_port: Port that socat maps to internally
+    """
+    
+    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
+        """Initialize the Docker browser strategy.
+        
+        Args:
+            config: Browser configuration including Docker-specific settings
+            logger: Logger for recording events and errors
+        """
+        super().__init__(config, logger)
+        
+        # Initialize Docker-specific attributes
+        self.docker_config = self.config.docker_config or DockerConfig()
+        self.container_id = None
+        self.container_name = f"crawl4ai-browser-{uuid.uuid4().hex[:8]}"
+        self.registry = DockerRegistry(self.docker_config.registry_file)
+        self.docker_utils = DockerUtils(logger)
+        self.chrome_process_id = None
+        self.socat_process_id = None
+        self.internal_cdp_port = 9222  # Chrome's internal CDP port
+        self.internal_mapped_port = 9223  # Port that socat maps to internally
+        self.shutting_down = False
+    
+    async def _generate_config_hash(self) -> str:
+        """Generate a hash of the configuration for container matching.
+        
+        Returns:
+            Hash string uniquely identifying this configuration
+        """
+        # Create a dict with the relevant parts of the config
+        config_dict = {
+            "image": self.docker_config.image,
+            "mode": self.docker_config.mode,
+            "browser_type": self.config.browser_type,
+            "headless": self.config.headless,
+        }
+        
+        # Add browser-specific config if in launch mode
+        if self.docker_config.mode == "launch":
+            config_dict.update({
+                "text_mode": self.config.text_mode,
+                "light_mode": self.config.light_mode,
+                "viewport_width": self.config.viewport_width,
+                "viewport_height": self.config.viewport_height,
+            })
+        
+        # Use the utility method to generate the hash
+        return self.docker_utils.generate_config_hash(config_dict)
+    
+    async def _get_or_create_cdp_url(self) -> str:
+        """Get CDP URL by either creating a new container or using an existing one.
+        
+        Returns:
+            CDP URL for connecting to the browser
+            
+        Raises:
+            Exception: If container creation or browser launch fails
+        """
+        # If CDP URL is explicitly provided, use it
+        if self.config.cdp_url:
+            return self.config.cdp_url
+        
+        # Ensure Docker image exists (will build if needed)
+        image_name = await self.docker_utils.ensure_docker_image_exists(
+            self.docker_config.image, 
+            self.docker_config.mode
+        )
+        
+        # Generate config hash for container matching
+        config_hash = await self._generate_config_hash()
+        
+        # Look for existing container with matching config
+        container_id = self.registry.find_container_by_config(config_hash, self.docker_utils)
+        
+        if container_id:
+            # Use existing container
+            self.container_id = container_id
+            host_port = self.registry.get_container_host_port(container_id)
+            if self.logger:
+                self.logger.info(f"Using existing Docker container: {container_id[:12]}", tag="DOCKER")
+        else:
+            # Get a port for the new container
+            host_port = self.docker_config.host_port or self.registry.get_next_available_port(self.docker_utils)
+            
+            # Prepare volumes list
+            volumes = list(self.docker_config.volumes)
+            
+            # Add user data directory if specified
+            if self.docker_config.user_data_dir:
+                # Ensure user data directory exists
+                os.makedirs(self.docker_config.user_data_dir, exist_ok=True)
+                volumes.append(f"{self.docker_config.user_data_dir}:{self.docker_config.container_user_data_dir}")
+                
+                # Update config user_data_dir to point to container path
+                self.config.user_data_dir = self.docker_config.container_user_data_dir
+            
+            # Create a new container
+            container_id = await self.docker_utils.create_container(
+                image_name=image_name,
+                host_port=host_port,
+                container_name=self.container_name,
+                volumes=volumes,
+                network=self.docker_config.network,
+                env_vars=self.docker_config.env_vars,
+                extra_args=self.docker_config.extra_args
+            )
+            
+            if not container_id:
+                raise Exception("Failed to create Docker container")
+            
+            self.container_id = container_id
+            
+            # Register the container
+            self.registry.register_container(container_id, host_port, config_hash)
+            
+            # Wait for container to be ready
+            await self.docker_utils.wait_for_container_ready(container_id)
+            
+            # Handle specific setup based on mode
+            if self.docker_config.mode == "launch":
+                # In launch mode, we need to start socat and Chrome
+                await self.docker_utils.start_socat_in_container(container_id)
+                
+                # Build browser arguments
+                browser_args = self._build_browser_args()
+                
+                # Launch Chrome
+                await self.docker_utils.launch_chrome_in_container(container_id, browser_args)
+                
+                # Get PIDs for later cleanup
+                self.chrome_process_id = await self.docker_utils.get_process_id_in_container(
+                    container_id, "chrome"
+                )
+                self.socat_process_id = await self.docker_utils.get_process_id_in_container(
+                    container_id, "socat"
+                )
+            
+            # Wait for CDP to be ready
+            await self.docker_utils.wait_for_cdp_ready(host_port)
+            
+            if self.logger:
+                self.logger.success(f"Docker container ready: {container_id[:12]} on port {host_port}", tag="DOCKER")
+        
+        # Return CDP URL
+        return f"http://localhost:{host_port}"
+    
+    def _build_browser_args(self) -> List[str]:
+        """Build Chrome command line arguments based on BrowserConfig.
+        
+        Returns:
+            List of command line arguments for Chrome
+        """
+        args = [
+            "--no-sandbox",
+            "--disable-gpu",
+            f"--remote-debugging-port={self.internal_cdp_port}",
+            "--remote-debugging-address=0.0.0.0",  # Allow external connections
+            "--disable-dev-shm-usage",
+        ]
+        
+        if self.config.headless:
+            args.append("--headless=new")
+            
+        if self.config.viewport_width and self.config.viewport_height:
+            args.append(f"--window-size={self.config.viewport_width},{self.config.viewport_height}")
+            
+        if self.config.user_agent:
+            args.append(f"--user-agent={self.config.user_agent}")
+            
+        if self.config.text_mode:
+            args.extend([
+                "--blink-settings=imagesEnabled=false",
+                "--disable-remote-fonts",
+                "--disable-images",
+                "--disable-javascript",
+            ])
+            
+        if self.config.light_mode:
+            # Import here to avoid circular import
+            from .utils import get_browser_disable_options
+            args.extend(get_browser_disable_options())
+            
+        if self.config.user_data_dir:
+            args.append(f"--user-data-dir={self.config.user_data_dir}")
+            
+        if self.config.extra_args:
+            args.extend(self.config.extra_args)
+            
+        return args
+    
+    async def close(self):
+        """Close the browser and clean up Docker container if needed."""
+        # Set shutting_down flag to prevent race conditions
+        self.shutting_down = True
+        
+        # Store state if needed before closing
+        if self.browser and self.docker_config.user_data_dir and self.docker_config.persistent:
+            for context in self.browser.contexts:
+                try:
+                    storage_path = os.path.join(self.docker_config.user_data_dir, "storage_state.json")
+                    await context.storage_state(path=storage_path)
+                    if self.logger:
+                        self.logger.debug("Persisted storage state before closing browser", tag="DOCKER")
+                except Exception as e:
+                    if self.logger:
+                        self.logger.warning(
+                            message="Failed to persist storage state: {error}",
+                            tag="DOCKER",
+                            params={"error": str(e)}
+                        )
+        
+        # Close browser connection (but not container)
+        if self.browser:
+            await self.browser.close()
+            self.browser = None
+        
+        # Only clean up container if not persistent
+        if self.container_id and not self.docker_config.persistent:
+            # Stop Chrome process in "launch" mode
+            if self.docker_config.mode == "launch" and self.chrome_process_id:
+                await self.docker_utils.stop_process_in_container(
+                    self.container_id, self.chrome_process_id
+                )
+            
+            # Stop socat process in "launch" mode
+            if self.docker_config.mode == "launch" and self.socat_process_id:
+                await self.docker_utils.stop_process_in_container(
+                    self.container_id, self.socat_process_id
+                )
+            
+            # Remove or stop container based on configuration
+            if self.docker_config.remove_on_exit:
+                await self.docker_utils.remove_container(self.container_id)
+                # Unregister from registry
+                self.registry.unregister_container(self.container_id)
+            else:
+                await self.docker_utils.stop_container(self.container_id)
+            
+            self.container_id = None
+        
+        # Close Playwright
+        if self.playwright:
+            await self.playwright.stop()
+            self.playwright = None
+        
+        self.shutting_down = False

crawl4ai/browser/docker_utils.py

@@ -8,14 +8,13 @@ import socket
 import subprocess
 from typing import Dict, List, Optional, Tuple, Union
 
-
 class DockerUtils:
     """Utility class for Docker operations in browser automation.
-
+    
     This class provides methods for managing Docker images, containers,
     and related operations needed for browser automation. It handles
     image building, container lifecycle, port management, and registry operations.
-
+    
     Attributes:
         DOCKER_FOLDER (str): Path to folder containing Docker files
         DOCKER_CONNECT_FILE (str): Path to Dockerfile for connect mode
@@ -25,38 +24,38 @@ class DockerUtils:
         DEFAULT_LAUNCH_IMAGE (str): Default image name for launch mode
         logger: Optional logger instance
     """
-
+    
     # File paths for Docker resources
     DOCKER_FOLDER = os.path.join(os.path.dirname(__file__), "docker")
     DOCKER_CONNECT_FILE = os.path.join(DOCKER_FOLDER, "connect.Dockerfile")
     DOCKER_LAUNCH_FILE = os.path.join(DOCKER_FOLDER, "launch.Dockerfile")
     DOCKER_START_SCRIPT = os.path.join(DOCKER_FOLDER, "start.sh")
-
+    
     # Default image names
     DEFAULT_CONNECT_IMAGE = "crawl4ai/browser-connect:latest"
     DEFAULT_LAUNCH_IMAGE = "crawl4ai/browser-launch:latest"
-
+    
     def __init__(self, logger=None):
         """Initialize Docker utilities.
-
+        
         Args:
             logger: Optional logger for recording operations
         """
         self.logger = logger
-
+    
     # Image Management Methods
-
+    
     async def check_image_exists(self, image_name: str) -> bool:
         """Check if a Docker image exists.
-
+        
         Args:
             image_name: Name of the Docker image to check
-
+            
         Returns:
             bool: True if the image exists, False otherwise
         """
         cmd = ["docker", "image", "inspect", image_name]
-
+        
         try:
             process = await asyncio.create_subprocess_exec(
                 *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
@@ -65,24 +64,18 @@ class DockerUtils:
             return process.returncode == 0
         except Exception as e:
             if self.logger:
-                self.logger.debug(
-                    f"Error checking if image exists: {str(e)}", tag="DOCKER"
-                )
+                self.logger.debug(f"Error checking if image exists: {str(e)}", tag="DOCKER")
             return False
-
-    async def build_docker_image(
-        self,
-        image_name: str,
-        dockerfile_path: str,
-        files_to_copy: Dict[str, str] = None,
-    ) -> bool:
+    
+    async def build_docker_image(self, image_name: str, dockerfile_path: str, 
+                              files_to_copy: Dict[str, str] = None) -> bool:
         """Build a Docker image from a Dockerfile.
-
+        
         Args:
             image_name: Name to give the built image
             dockerfile_path: Path to the Dockerfile
             files_to_copy: Dict of {dest_name: source_path} for files to copy to build context
-
+            
         Returns:
             bool: True if image was built successfully, False otherwise
         """
@@ -90,119 +83,103 @@ class DockerUtils:
         with tempfile.TemporaryDirectory() as temp_dir:
             # Copy the Dockerfile
             shutil.copy(dockerfile_path, os.path.join(temp_dir, "Dockerfile"))
-
+            
             # Copy any additional files needed
             if files_to_copy:
                 for dest_name, source_path in files_to_copy.items():
                     shutil.copy(source_path, os.path.join(temp_dir, dest_name))
-
+            
             # Build the image
-            cmd = ["docker", "build", "-t", image_name, temp_dir]
-
+            cmd = [
+                "docker", "build",
+                "-t", image_name,
+                temp_dir
+            ]
+            
             if self.logger:
-                self.logger.debug(
-                    f"Building Docker image with command: {' '.join(cmd)}", tag="DOCKER"
-                )
-
+                self.logger.debug(f"Building Docker image with command: {' '.join(cmd)}", tag="DOCKER")
+            
             process = await asyncio.create_subprocess_exec(
                 *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
             )
             stdout, stderr = await process.communicate()
-
+            
             if process.returncode != 0:
                 if self.logger:
                     self.logger.error(
                         message="Failed to build Docker image: {error}",
                         tag="DOCKER",
-                        params={"error": stderr.decode()},
+                        params={"error": stderr.decode()}
                     )
                 return False
-
+            
             if self.logger:
-                self.logger.success(
-                    f"Successfully built Docker image: {image_name}", tag="DOCKER"
-                )
+                self.logger.success(f"Successfully built Docker image: {image_name}", tag="DOCKER")
             return True
-
-    async def ensure_docker_image_exists(
-        self, image_name: str, mode: str = "connect"
-    ) -> str:
+    
+    async def ensure_docker_image_exists(self, image_name: str, mode: str = "connect") -> str:
         """Ensure the required Docker image exists, creating it if necessary.
-
+        
         Args:
             image_name: Name of the Docker image
             mode: Either "connect" or "launch" to determine which image to build
-
+            
         Returns:
             str: Name of the available Docker image
-
+            
         Raises:
             Exception: If image doesn't exist and can't be built
         """
         # If image name is not specified, use default based on mode
         if not image_name:
-            image_name = (
-                self.DEFAULT_CONNECT_IMAGE
-                if mode == "connect"
-                else self.DEFAULT_LAUNCH_IMAGE
-            )
-
+            image_name = self.DEFAULT_CONNECT_IMAGE if mode == "connect" else self.DEFAULT_LAUNCH_IMAGE
+        
         # Check if the image already exists
         if await self.check_image_exists(image_name):
             if self.logger:
-                self.logger.debug(
-                    f"Docker image {image_name} already exists", tag="DOCKER"
-                )
+                self.logger.debug(f"Docker image {image_name} already exists", tag="DOCKER")
             return image_name
-
+        
         # If we're using a custom image that doesn't exist, warn and fail
-        if (
-            image_name != self.DEFAULT_CONNECT_IMAGE
-            and image_name != self.DEFAULT_LAUNCH_IMAGE
-        ):
+        if (image_name != self.DEFAULT_CONNECT_IMAGE and image_name != self.DEFAULT_LAUNCH_IMAGE):
             if self.logger:
                 self.logger.warning(
                     f"Custom Docker image {image_name} not found and cannot be automatically created",
-                    tag="DOCKER",
+                    tag="DOCKER"
                 )
             raise Exception(f"Docker image {image_name} not found")
-
+        
         # Build the appropriate default image
         if self.logger:
-            self.logger.info(
-                f"Docker image {image_name} not found, creating it now...", tag="DOCKER"
-            )
-
+            self.logger.info(f"Docker image {image_name} not found, creating it now...", tag="DOCKER")
+        
         if mode == "connect":
             success = await self.build_docker_image(
-                image_name,
-                self.DOCKER_CONNECT_FILE,
-                {"start.sh": self.DOCKER_START_SCRIPT},
+                image_name, 
+                self.DOCKER_CONNECT_FILE, 
+                {"start.sh": self.DOCKER_START_SCRIPT}
             )
         else:
-            success = await self.build_docker_image(image_name, self.DOCKER_LAUNCH_FILE)
-
+            success = await self.build_docker_image(
+                image_name, 
+                self.DOCKER_LAUNCH_FILE
+            )
+        
         if not success:
             raise Exception(f"Failed to create Docker image {image_name}")
-
+        
         return image_name
-
+    
     # Container Management Methods
-
-    async def create_container(
-        self,
-        image_name: str,
-        host_port: int,
-        container_name: Optional[str] = None,
-        volumes: List[str] = None,
-        network: Optional[str] = None,
-        env_vars: Dict[str, str] = None,
-        cpu_limit: float = 1.0,
-        memory_limit: str = "1.5g",
-        extra_args: List[str] = None,
-    ) -> Optional[str]:
+    
+    async def create_container(self, image_name: str, host_port: int, 
+                            container_name: Optional[str] = None,
+                            volumes: List[str] = None,
+                            network: Optional[str] = None,
+                            env_vars: Dict[str, str] = None,
+                            extra_args: List[str] = None) -> Optional[str]:
         """Create a new Docker container.
-
+        
         Args:
             image_name: Docker image to use
             host_port: Port on host to map to container port 9223
@@ -210,134 +187,111 @@ class DockerUtils:
             volumes: List of volume mappings (e.g., ["host_path:container_path"])
             network: Optional Docker network to use
             env_vars: Dictionary of environment variables
-            cpu_limit: CPU limit for the container
-            memory_limit: Memory limit for the container
             extra_args: Additional docker run arguments
-
+            
         Returns:
             str: Container ID if successful, None otherwise
         """
         # Prepare container command
         cmd = [
-            "docker",
-            "run",
+            "docker", "run",
             "--detach",
         ]
-
+        
         # Add container name if specified
         if container_name:
             cmd.extend(["--name", container_name])
-
+        
         # Add port mapping
         cmd.extend(["-p", f"{host_port}:9223"])
-
+        
         # Add volumes
         if volumes:
             for volume in volumes:
                 cmd.extend(["-v", volume])
-
+        
         # Add network if specified
         if network:
             cmd.extend(["--network", network])
-
+        
         # Add environment variables
         if env_vars:
             for key, value in env_vars.items():
                 cmd.extend(["-e", f"{key}={value}"])
-
-        # Add CPU and memory limits
-        if cpu_limit:
-            cmd.extend(["--cpus", str(cpu_limit)])
-        if memory_limit:
-            cmd.extend(["--memory", memory_limit])
-            cmd.extend(["--memory-swap", memory_limit])
-        if self.logger:
-            self.logger.debug(
-                f"Setting CPU limit: {cpu_limit}, Memory limit: {memory_limit}",
-                tag="DOCKER",
-            )
-
+        
         # Add extra args
         if extra_args:
             cmd.extend(extra_args)
-
+        
         # Add image
         cmd.append(image_name)
-
+        
         if self.logger:
-            self.logger.debug(
-                f"Creating Docker container with command: {' '.join(cmd)}", tag="DOCKER"
-            )
-
+            self.logger.debug(f"Creating Docker container with command: {' '.join(cmd)}", tag="DOCKER")
+        
         # Run docker command
         try:
             process = await asyncio.create_subprocess_exec(
                 *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
             )
             stdout, stderr = await process.communicate()
-
+            
             if process.returncode != 0:
                 if self.logger:
                     self.logger.error(
                         message="Failed to create Docker container: {error}",
                         tag="DOCKER",
-                        params={"error": stderr.decode()},
+                        params={"error": stderr.decode()}
                     )
                 return None
-
+            
             # Get container ID
             container_id = stdout.decode().strip()
-
+            
             if self.logger:
-                self.logger.success(
-                    f"Created Docker container: {container_id[:12]}", tag="DOCKER"
-                )
-
+                self.logger.success(f"Created Docker container: {container_id[:12]}", tag="DOCKER")
+            
             return container_id
-
+            
         except Exception as e:
             if self.logger:
                 self.logger.error(
                     message="Error creating Docker container: {error}",
                     tag="DOCKER",
-                    params={"error": str(e)},
+                    params={"error": str(e)}
                 )
             return None
-
+    
     async def is_container_running(self, container_id: str) -> bool:
         """Check if a container is running.
-
+        
         Args:
             container_id: ID of the container to check
-
+            
         Returns:
             bool: True if the container is running, False otherwise
         """
         cmd = ["docker", "inspect", "--format", "{{.State.Running}}", container_id]
-
+        
         try:
             process = await asyncio.create_subprocess_exec(
                 *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
             )
             stdout, _ = await process.communicate()
-
+            
             return process.returncode == 0 and stdout.decode().strip() == "true"
         except Exception as e:
             if self.logger:
-                self.logger.debug(
-                    f"Error checking if container is running: {str(e)}", tag="DOCKER"
-                )
+                self.logger.debug(f"Error checking if container is running: {str(e)}", tag="DOCKER")
             return False
-
-    async def wait_for_container_ready(
-        self, container_id: str, timeout: int = 30
-    ) -> bool:
+    
+    async def wait_for_container_ready(self, container_id: str, timeout: int = 30) -> bool:
         """Wait for the container to be in running state.
-
+        
         Args:
             container_id: ID of the container to wait for
             timeout: Maximum time to wait in seconds
-
+            
         Returns:
             bool: True if container is ready, False if timeout occurred
         """
@@ -345,51 +299,46 @@ class DockerUtils:
             if await self.is_container_running(container_id):
                 return True
             await asyncio.sleep(1)
-
+        
         if self.logger:
-            self.logger.warning(
-                f"Container {container_id[:12]} not ready after {timeout}s timeout",
-                tag="DOCKER",
-            )
+            self.logger.warning(f"Container {container_id[:12]} not ready after {timeout}s timeout", tag="DOCKER")
         return False
-
+    
     async def stop_container(self, container_id: str) -> bool:
         """Stop a Docker container.
-
+        
         Args:
             container_id: ID of the container to stop
-
+            
         Returns:
             bool: True if stopped successfully, False otherwise
         """
         cmd = ["docker", "stop", container_id]
-
+        
         try:
             process = await asyncio.create_subprocess_exec(*cmd)
             await process.communicate()
-
+            
             if self.logger:
-                self.logger.debug(
-                    f"Stopped container: {container_id[:12]}", tag="DOCKER"
-                )
-
+                self.logger.debug(f"Stopped container: {container_id[:12]}", tag="DOCKER")
+                
             return process.returncode == 0
         except Exception as e:
             if self.logger:
                 self.logger.warning(
                     message="Failed to stop container: {error}",
                     tag="DOCKER",
-                    params={"error": str(e)},
+                    params={"error": str(e)}
                 )
             return False
-
+    
     async def remove_container(self, container_id: str, force: bool = True) -> bool:
         """Remove a Docker container.
-
+        
         Args:
             container_id: ID of the container to remove
             force: Whether to force removal
-
+            
         Returns:
             bool: True if removed successfully, False otherwise
         """
@@ -397,38 +346,35 @@ class DockerUtils:
         if force:
             cmd.append("-f")
         cmd.append(container_id)
-
+        
         try:
             process = await asyncio.create_subprocess_exec(*cmd)
             await process.communicate()
-
+            
             if self.logger:
-                self.logger.debug(
-                    f"Removed container: {container_id[:12]}", tag="DOCKER"
-                )
-
+                self.logger.debug(f"Removed container: {container_id[:12]}", tag="DOCKER")
+                
             return process.returncode == 0
         except Exception as e:
             if self.logger:
                 self.logger.warning(
                     message="Failed to remove container: {error}",
                     tag="DOCKER",
-                    params={"error": str(e)},
+                    params={"error": str(e)}
                 )
             return False
-
+    
     # Container Command Execution Methods
-
-    async def exec_in_container(
-        self, container_id: str, command: List[str], detach: bool = False
-    ) -> Tuple[int, str, str]:
+    
+    async def exec_in_container(self, container_id: str, command: List[str], 
+                             detach: bool = False) -> Tuple[int, str, str]:
         """Execute a command in a running container.
-
+        
         Args:
             container_id: ID of the container
             command: Command to execute as a list of strings
             detach: Whether to run the command in detached mode
-
+            
         Returns:
             Tuple of (return_code, stdout, stderr)
         """
@@ -437,206 +383,181 @@ class DockerUtils:
             cmd.append("-d")
         cmd.append(container_id)
         cmd.extend(command)
-
+        
         try:
             process = await asyncio.create_subprocess_exec(
                 *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
             )
             stdout, stderr = await process.communicate()
-
+            
             return process.returncode, stdout.decode(), stderr.decode()
         except Exception as e:
             if self.logger:
                 self.logger.error(
                     message="Error executing command in container: {error}",
                     tag="DOCKER",
-                    params={"error": str(e)},
+                    params={"error": str(e)}
                 )
             return -1, "", str(e)
-
+    
     async def start_socat_in_container(self, container_id: str) -> bool:
         """Start socat in the container to map port 9222 to 9223.
-
+        
         Args:
             container_id: ID of the container
-
+            
         Returns:
             bool: True if socat started successfully, False otherwise
         """
         # Command to run socat as a background process
         cmd = ["socat", "TCP-LISTEN:9223,fork", "TCP:localhost:9222"]
-
-        returncode, _, stderr = await self.exec_in_container(
-            container_id, cmd, detach=True
-        )
-
+        
+        returncode, _, stderr = await self.exec_in_container(container_id, cmd, detach=True)
+        
         if returncode != 0:
             if self.logger:
                 self.logger.error(
                     message="Failed to start socat in container: {error}",
                     tag="DOCKER",
-                    params={"error": stderr},
+                    params={"error": stderr}
                 )
             return False
-
+            
         if self.logger:
-            self.logger.debug(
-                f"Started socat in container: {container_id[:12]}", tag="DOCKER"
-            )
-
+            self.logger.debug(f"Started socat in container: {container_id[:12]}", tag="DOCKER")
+        
         # Wait a moment for socat to start
         await asyncio.sleep(1)
         return True
-
-    async def launch_chrome_in_container(
-        self, container_id: str, browser_args: List[str]
-    ) -> bool:
+    
+    async def launch_chrome_in_container(self, container_id: str, browser_args: List[str]) -> bool:
         """Launch Chrome inside the container with specified arguments.
-
+        
         Args:
             container_id: ID of the container
             browser_args: Chrome command line arguments
-
+            
         Returns:
             bool: True if Chrome started successfully, False otherwise
         """
         # Build Chrome command
-        chrome_cmd = ["chromium"]
+        chrome_cmd = ["google-chrome"]
         chrome_cmd.extend(browser_args)
-
-        returncode, _, stderr = await self.exec_in_container(
-            container_id, chrome_cmd, detach=True
-        )
-
+        
+        returncode, _, stderr = await self.exec_in_container(container_id, chrome_cmd, detach=True)
+        
         if returncode != 0:
             if self.logger:
                 self.logger.error(
                     message="Failed to launch Chrome in container: {error}",
                     tag="DOCKER",
-                    params={"error": stderr},
+                    params={"error": stderr}
                 )
             return False
-
+            
         if self.logger:
-            self.logger.debug(
-                f"Launched Chrome in container: {container_id[:12]}", tag="DOCKER"
-            )
-
+            self.logger.debug(f"Launched Chrome in container: {container_id[:12]}", tag="DOCKER")
+        
         return True
-
-    async def get_process_id_in_container(
-        self, container_id: str, process_name: str
-    ) -> Optional[int]:
+    
+    async def get_process_id_in_container(self, container_id: str, process_name: str) -> Optional[int]:
         """Get the process ID for a process in the container.
-
+        
         Args:
             container_id: ID of the container
             process_name: Name pattern to search for
-
+            
         Returns:
             int: Process ID if found, None otherwise
         """
         cmd = ["pgrep", "-f", process_name]
-
+        
         returncode, stdout, _ = await self.exec_in_container(container_id, cmd)
-
+        
         if returncode == 0 and stdout.strip():
             pid = int(stdout.strip().split("\n")[0])
             return pid
-
+        
         return None
-
+    
     async def stop_process_in_container(self, container_id: str, pid: int) -> bool:
         """Stop a process in the container by PID.
-
+        
         Args:
             container_id: ID of the container
             pid: Process ID to stop
-
+            
         Returns:
             bool: True if process was stopped, False otherwise
         """
         cmd = ["kill", "-TERM", str(pid)]
-
+        
         returncode, _, stderr = await self.exec_in_container(container_id, cmd)
-
+        
         if returncode != 0:
             if self.logger:
                 self.logger.warning(
                     message="Failed to stop process in container: {error}",
                     tag="DOCKER",
-                    params={"error": stderr},
+                    params={"error": stderr}
                 )
             return False
-
+            
         if self.logger:
-            self.logger.debug(
-                f"Stopped process {pid} in container: {container_id[:12]}", tag="DOCKER"
-            )
-
+            self.logger.debug(f"Stopped process {pid} in container: {container_id[:12]}", tag="DOCKER")
+        
         return True
-
+    
     # Network and Port Methods
-
-    async def wait_for_cdp_ready(self, host_port: int, timeout: int = 10) -> dict:
+    
+    async def wait_for_cdp_ready(self, host_port: int, timeout: int = 30) -> bool:
         """Wait for the CDP endpoint to be ready.
-
+        
         Args:
             host_port: Port to check for CDP endpoint
             timeout: Maximum time to wait in seconds
-
+            
         Returns:
-            dict: CDP JSON config if ready, None if timeout occurred
+            bool: True if CDP endpoint is ready, False if timeout occurred
         """
         import aiohttp
-
+        
         url = f"http://localhost:{host_port}/json/version"
-
+        
         for _ in range(timeout):
             try:
                 async with aiohttp.ClientSession() as session:
                     async with session.get(url, timeout=1) as response:
                         if response.status == 200:
                             if self.logger:
-                                self.logger.debug(
-                                    f"CDP endpoint ready on port {host_port}",
-                                    tag="DOCKER",
-                                )
-                            cdp_json_config = await response.json()
-                            if self.logger:
-                                self.logger.debug(
-                                    f"CDP JSON config: {cdp_json_config}", tag="DOCKER"
-                                )
-                            return cdp_json_config
+                                self.logger.debug(f"CDP endpoint ready on port {host_port}", tag="DOCKER")
+                            return True
             except Exception:
                 pass
             await asyncio.sleep(1)
-
+        
         if self.logger:
-            self.logger.warning(
-                f"CDP endpoint not ready on port {host_port} after {timeout}s timeout",
-                tag="DOCKER",
-            )
-        return None
-
+            self.logger.warning(f"CDP endpoint not ready on port {host_port} after {timeout}s timeout", tag="DOCKER")
+        return False
+    
     def is_port_in_use(self, port: int) -> bool:
         """Check if a port is already in use on the host.
-
+        
         Args:
             port: Port number to check
-
+            
         Returns:
             bool: True if port is in use, False otherwise
         """
         with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-            return s.connect_ex(("localhost", port)) == 0
-
+            return s.connect_ex(('localhost', port)) == 0
+    
     def get_next_available_port(self, start_port: int = 9223) -> int:
         """Get the next available port starting from a given port.
-
+        
         Args:
             start_port: Port number to start checking from
-
+            
         Returns:
             int: First available port number
         """
@@ -644,18 +565,18 @@ class DockerUtils:
         while self.is_port_in_use(port):
             port += 1
         return port
-
+    
     # Configuration Hash Methods
-
+    
     def generate_config_hash(self, config_dict: Dict) -> str:
         """Generate a hash of the configuration for container matching.
-
+        
         Args:
             config_dict: Dictionary of configuration parameters
-
+            
         Returns:
             str: Hash string uniquely identifying this configuration
         """
         # Convert to canonical JSON string and hash
         config_json = json.dumps(config_dict, sort_keys=True)
-        return hashlib.sha256(config_json.encode()).hexdigest()
+        return hashlib.sha256(config_json.encode()).hexdigest()

crawl4ai/browser/manager copy.py

@@ -1,177 +0,0 @@
-"""Browser manager module for Crawl4AI.
-
-This module provides a central browser management class that uses the
-strategy pattern internally while maintaining the existing API.
-It also implements a page pooling mechanism for improved performance.
-"""
-
-from typing import Optional, Tuple, List
-
-from playwright.async_api import Page, BrowserContext
-
-from ..async_logger import AsyncLogger
-from ..async_configs import BrowserConfig, CrawlerRunConfig
-
-from .strategies import (
-    BaseBrowserStrategy,
-    PlaywrightBrowserStrategy,
-    CDPBrowserStrategy,
-    BuiltinBrowserStrategy,
-    DockerBrowserStrategy
-)
-
-class BrowserManager:
-    """Main interface for browser management in Crawl4AI.
-    
-    This class maintains backward compatibility with the existing implementation
-    while using the strategy pattern internally for different browser types.
-    
-    Attributes:
-        config (BrowserConfig): Configuration object containing all browser settings
-        logger: Logger instance for recording events and errors
-        browser: The browser instance
-        default_context: The default browser context
-        managed_browser: The managed browser instance
-        playwright: The Playwright instance
-        sessions: Dictionary to store session information
-        session_ttl: Session timeout in seconds
-    """
-    
-    def __init__(self, browser_config: Optional[BrowserConfig] = None, logger: Optional[AsyncLogger] = None):
-        """Initialize the BrowserManager with a browser configuration.
-        
-        Args:
-            browser_config: Configuration object containing all browser settings
-            logger: Logger instance for recording events and errors
-        """
-        self.config = browser_config or BrowserConfig()
-        self.logger = logger
-        
-        # Create strategy based on configuration
-        self.strategy = self._create_strategy()
-        
-        # Initialize state variables for compatibility with existing code
-        self.browser = None
-        self.default_context = None
-        self.managed_browser = None
-        self.playwright = None
-        
-        # For session management (from existing implementation)
-        self.sessions = {}
-        self.session_ttl = 1800  # 30 minutes
-    
-    def _create_strategy(self) -> BaseBrowserStrategy:
-        """Create appropriate browser strategy based on configuration.
-        
-        Returns:
-            BaseBrowserStrategy: The selected browser strategy
-        """
-        if self.config.browser_mode == "builtin":
-            return BuiltinBrowserStrategy(self.config, self.logger)
-        elif self.config.browser_mode == "docker":
-            if DockerBrowserStrategy is None:
-                if self.logger:
-                    self.logger.error(
-                        "Docker browser strategy requested but not available. "
-                        "Falling back to PlaywrightBrowserStrategy.",
-                        tag="BROWSER"
-                    )
-                return PlaywrightBrowserStrategy(self.config, self.logger)
-            return DockerBrowserStrategy(self.config, self.logger)
-        elif self.config.browser_mode == "cdp" or self.config.cdp_url or self.config.use_managed_browser:
-            return CDPBrowserStrategy(self.config, self.logger)
-        else:
-            return PlaywrightBrowserStrategy(self.config, self.logger)
-    
-    async def start(self):
-        """Start the browser instance and set up the default context.
-        
-        Returns:
-            self: For method chaining
-        """
-        # Start the strategy
-        await self.strategy.start()
-        
-        # Update legacy references
-        self.browser = self.strategy.browser
-        self.default_context = self.strategy.default_context
-        
-        # Set browser process reference (for CDP strategy)
-        if hasattr(self.strategy, 'browser_process'):
-            self.managed_browser = self.strategy
-        
-        # Set Playwright reference
-        self.playwright = self.strategy.playwright
-        
-        # Sync sessions if needed
-        if hasattr(self.strategy, 'sessions'):
-            self.sessions = self.strategy.sessions
-            self.session_ttl = self.strategy.session_ttl
-        
-        return self
-    
-    async def get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
-        """Get a page for the given configuration.
-        
-        Args:
-            crawlerRunConfig: Configuration object for the crawler run
-            
-        Returns:
-            Tuple of (Page, BrowserContext)
-        """
-        # Delegate to strategy
-        page, context = await self.strategy.get_page(crawlerRunConfig)
-        
-        # Sync sessions if needed
-        if hasattr(self.strategy, 'sessions'):
-            self.sessions = self.strategy.sessions
-        
-        return page, context
-        
-    async def get_pages(self, crawlerRunConfig: CrawlerRunConfig, count: int = 1) -> List[Tuple[Page, BrowserContext]]:
-        """Get multiple pages with the same configuration.
-        
-        This method efficiently creates multiple browser pages using the same configuration,
-        which is useful for parallel crawling of multiple URLs.
-        
-        Args:
-            crawlerRunConfig: Configuration for the pages
-            count: Number of pages to create
-            
-        Returns:
-            List of (Page, Context) tuples
-        """
-        # Delegate to strategy
-        pages = await self.strategy.get_pages(crawlerRunConfig, count)
-        
-        # Sync sessions if needed
-        if hasattr(self.strategy, 'sessions'):
-            self.sessions = self.strategy.sessions
-            
-        return pages
-    
-    # Just for legacy compatibility
-    async def kill_session(self, session_id: str):
-        """Kill a browser session and clean up resources.
-        
-        Args:
-            session_id: The session ID to kill
-        """
-        # Handle kill_session via our strategy if it supports it
-        await self.strategy.kill_session(session_id)
-
-        # sync sessions if needed
-        if hasattr(self.strategy, 'sessions'):
-            self.sessions = self.strategy.sessions
-    
-    async def close(self):
-        """Close the browser and clean up resources."""
-        # Delegate to strategy
-        await self.strategy.close()
-        
-        # Reset legacy references
-        self.browser = None
-        self.default_context = None
-        self.managed_browser = None
-        self.playwright = None
-        self.sessions = {}

crawl4ai/browser/manager.py

@@ -2,15 +2,12 @@
 
 This module provides a central browser management class that uses the
 strategy pattern internally while maintaining the existing API.
-It also implements browser pooling for improved performance.
+It also implements a page pooling mechanism for improved performance.
 """
 
 import asyncio
-import hashlib
-import json
-import math
-from enum import Enum
-from typing import Dict, List, Optional, Tuple, Any
+import time
+from typing import Optional, Tuple, List
 
 from playwright.async_api import Page, BrowserContext
 
@@ -21,115 +18,64 @@ from .strategies import (
     BaseBrowserStrategy,
     PlaywrightBrowserStrategy,
     CDPBrowserStrategy,
-    BuiltinBrowserStrategy,
-    DockerBrowserStrategy
+    BuiltinBrowserStrategy
 )
 
-class UnavailableBehavior(Enum):
-    """Behavior when no browser is available."""
-    ON_DEMAND = "on_demand"  # Create new browser on demand
-    PENDING = "pending"      # Wait until a browser is available
-    EXCEPTION = "exception"  # Raise an exception
-
+# Import DockerBrowserStrategy if available
+try:
+    from .docker_strategy import DockerBrowserStrategy
+except ImportError:
+    DockerBrowserStrategy = None
 
 class BrowserManager:
-    """Main interface for browser management and pooling in Crawl4AI.
+    """Main interface for browser management in Crawl4AI.
     
     This class maintains backward compatibility with the existing implementation
     while using the strategy pattern internally for different browser types.
-    It also implements browser pooling for improved performance.
     
     Attributes:
-        config (BrowserConfig): Default configuration object for browsers
-        logger (AsyncLogger): Logger instance for recording events and errors
-        browser_pool (Dict): Dictionary to store browser instances by configuration
-        browser_in_use (Dict): Dictionary to track which browsers are in use
-        request_queues (Dict): Queues for pending requests by configuration
-        unavailable_behavior (UnavailableBehavior): Behavior when no browser is available
+        config (BrowserConfig): Configuration object containing all browser settings
+        logger: Logger instance for recording events and errors
+        browser: The browser instance
+        default_context: The default browser context
+        managed_browser: The managed browser instance
+        playwright: The Playwright instance
+        sessions: Dictionary to store session information
+        session_ttl: Session timeout in seconds
     """
     
-    def __init__(
-        self, 
-        browser_config: Optional[BrowserConfig] = None, 
-        logger: Optional[AsyncLogger] = None,
-        unavailable_behavior: UnavailableBehavior = UnavailableBehavior.EXCEPTION,
-        max_browsers_per_config: int = 10,
-        max_pages_per_browser: int = 5
-        ):
+    def __init__(self, browser_config: Optional[BrowserConfig] = None, logger: Optional[AsyncLogger] = None):
         """Initialize the BrowserManager with a browser configuration.
         
         Args:
             browser_config: Configuration object containing all browser settings
             logger: Logger instance for recording events and errors
-            unavailable_behavior: Behavior when no browser is available
-            max_browsers_per_config: Maximum number of browsers per configuration
-            max_pages_per_browser: Maximum number of pages per browser
         """
         self.config = browser_config or BrowserConfig()
         self.logger = logger
-        self.unavailable_behavior = unavailable_behavior
-        self.max_browsers_per_config = max_browsers_per_config
-        self.max_pages_per_browser = max_pages_per_browser
         
-        # Browser pool management
-        self.browser_pool = {}            # config_hash -> list of browser strategies
-        self.browser_in_use = {}          # strategy instance -> Boolean
-        self.request_queues = {}          # config_hash -> asyncio.Queue()
-        self._browser_locks = {}          # config_hash -> asyncio.Lock()
-        self._browser_pool_lock = asyncio.Lock()  # Global lock for pool modifications
+        # Create strategy based on configuration
+        self._strategy = self._create_strategy()
         
-        # Page pool management
-        self.page_pool = {}  # (browser_config_hash, crawler_config_hash) -> list of (page, context, strategy)
-        self._page_pool_lock = asyncio.Lock()
-            
-        self.browser_page_counts = {}  # strategy instance -> current page count
-        self._page_count_lock = asyncio.Lock()  # Lock for thread-safe access to page counts
-
-        # For session management (from existing implementation)
-        self.sessions = {}
-        self.session_ttl = 1800  # 30 minutes
-
-        # For legacy compatibility
+        # Initialize state variables for compatibility with existing code
         self.browser = None
         self.default_context = None
         self.managed_browser = None
         self.playwright = None
-        self.strategy = None
+        
+        # For session management (from existing implementation)
+        self.sessions = {}
+        self.session_ttl = 1800  # 30 minutes
     
-    def _create_browser_config_hash(self, browser_config: BrowserConfig) -> str:
-        """Create a hash of the browser configuration for browser pooling.
-        
-        Args:
-            browser_config: Browser configuration
-            
-        Returns:
-            str: Hash of the browser configuration
-        """
-        # Convert config to dictionary, excluding any callable objects
-        config_dict = browser_config.__dict__.copy()
-        for key in list(config_dict.keys()):
-            if callable(config_dict[key]):
-                del config_dict[key]
-        
-        # Convert to canonical JSON string
-        config_json = json.dumps(config_dict, sort_keys=True, default=str)
-        
-        # Hash the JSON
-        config_hash = hashlib.sha256(config_json.encode()).hexdigest()
-        return config_hash
-    
-    def _create_strategy(self, browser_config: BrowserConfig) -> BaseBrowserStrategy:
+    def _create_strategy(self) -> BaseBrowserStrategy:
         """Create appropriate browser strategy based on configuration.
         
-        Args:
-            browser_config: Browser configuration
-            
         Returns:
             BaseBrowserStrategy: The selected browser strategy
         """
-        if browser_config.browser_mode == "builtin":
-            return BuiltinBrowserStrategy(browser_config, self.logger)
-        elif browser_config.browser_mode == "docker":
+        if self.config.browser_mode == "builtin":
+            return BuiltinBrowserStrategy(self.config, self.logger)
+        elif self.config.browser_mode == "docker":
             if DockerBrowserStrategy is None:
                 if self.logger:
                     self.logger.error(
@@ -137,717 +83,122 @@ class BrowserManager:
                         "Falling back to PlaywrightBrowserStrategy.",
                         tag="BROWSER"
                     )
-                return PlaywrightBrowserStrategy(browser_config, self.logger)
-            return DockerBrowserStrategy(browser_config, self.logger)
-        elif browser_config.browser_mode == "cdp" or browser_config.cdp_url or browser_config.use_managed_browser:
-            return CDPBrowserStrategy(browser_config, self.logger)
+                return PlaywrightBrowserStrategy(self.config, self.logger)
+            return DockerBrowserStrategy(self.config, self.logger)
+        elif self.config.cdp_url or self.config.use_managed_browser:
+            return CDPBrowserStrategy(self.config, self.logger)
         else:
-            return PlaywrightBrowserStrategy(browser_config, self.logger)
+            return PlaywrightBrowserStrategy(self.config, self.logger)
     
-    async def initialize_pool(
-        self, 
-        browser_configs: List[BrowserConfig] = None,
-        browsers_per_config: int = 1,
-        page_configs: Optional[List[Tuple[BrowserConfig, CrawlerRunConfig, int]]] = None
-    ):
-        """Initialize the browser pool with multiple browser configurations.
+    async def start(self):
+        """Start the browser instance and set up the default context.
         
-        Args:
-            browser_configs: List of browser configurations to initialize
-            browsers_per_config: Number of browser instances per configuration
-            page_configs: Optional list of (browser_config, crawler_run_config, count) tuples
-                for pre-warming pages
-                
         Returns:
             self: For method chaining
         """
-        if not browser_configs:
-            browser_configs = [self.config]
-        
-        # Calculate how many browsers we'll need based on page_configs
-        browsers_needed = {}
-        if page_configs:
-            for browser_config, _, page_count in page_configs:
-                config_hash = self._create_browser_config_hash(browser_config)
-                # Calculate browsers based on max_pages_per_browser
-                browsers_needed_for_config = math.ceil(page_count / self.max_pages_per_browser)
-                browsers_needed[config_hash] = max(
-                    browsers_needed.get(config_hash, 0),
-                    browsers_needed_for_config
-                )
-        
-        # Adjust browsers_per_config if needed to ensure enough capacity
-        config_browsers_needed = {}
-        for browser_config in browser_configs:
-            config_hash = self._create_browser_config_hash(browser_config)
-            
-            # Estimate browsers needed based on page requirements
-            browsers_for_config = browsers_per_config
-            if config_hash in browsers_needed:
-                browsers_for_config = max(browsers_for_config, browsers_needed[config_hash])
-            
-            config_browsers_needed[config_hash] = browsers_for_config
-            
-            # Update max_browsers_per_config if needed
-            if browsers_for_config > self.max_browsers_per_config:
-                self.max_browsers_per_config = browsers_for_config
-                if self.logger:
-                    self.logger.info(
-                        f"Increased max_browsers_per_config to {browsers_for_config} to accommodate page requirements",
-                        tag="POOL"
-                    )
-        
-        # Initialize locks and queues for each config
-        async with self._browser_pool_lock:
-            for browser_config in browser_configs:
-                config_hash = self._create_browser_config_hash(browser_config)
-                
-                # Initialize lock for this config if needed
-                if config_hash not in self._browser_locks:
-                    self._browser_locks[config_hash] = asyncio.Lock()
-                
-                # Initialize queue for this config if needed
-                if config_hash not in self.request_queues:
-                    self.request_queues[config_hash] = asyncio.Queue()
-                
-                # Initialize pool for this config if needed
-                if config_hash not in self.browser_pool:
-                    self.browser_pool[config_hash] = []
-        
-        # Create browser instances for each configuration in parallel
-        browser_tasks = []
-        
-        for browser_config in browser_configs:
-            config_hash = self._create_browser_config_hash(browser_config)
-            browsers_to_create = config_browsers_needed.get(
-                config_hash, 
-                browsers_per_config
-            ) - len(self.browser_pool.get(config_hash, []))
-            
-            if browsers_to_create <= 0:
-                continue
-                
-            for _ in range(browsers_to_create):
-                # Create a task for each browser initialization
-                task = self._create_and_add_browser(browser_config, config_hash)
-                browser_tasks.append(task)
-        
-        # Wait for all browser initializations to complete
-        if browser_tasks:
-            if self.logger:
-                self.logger.info(f"Initializing {len(browser_tasks)} browsers in parallel...", tag="POOL")
-            await asyncio.gather(*browser_tasks)
-        
-        # Pre-warm pages if requested
-        if page_configs:
-            page_tasks = []
-            for browser_config, crawler_run_config, count in page_configs:
-                task = self._prewarm_pages(browser_config, crawler_run_config, count)
-                page_tasks.append(task)
-            
-            if page_tasks:
-                if self.logger:
-                    self.logger.info(f"Pre-warming pages with {len(page_tasks)} configurations...", tag="POOL")
-                await asyncio.gather(*page_tasks)
+        # Start the strategy
+        await self._strategy.start()
         
         # Update legacy references
-        if self.browser_pool and next(iter(self.browser_pool.values()), []):
-            strategy = next(iter(self.browser_pool.values()))[0]
-            self.strategy = strategy
-            self.browser = strategy.browser
-            self.default_context = strategy.default_context
-            self.playwright = strategy.playwright
+        self.browser = self._strategy.browser
+        self.default_context = self._strategy.default_context
+        
+        # Set browser process reference (for CDP strategy)
+        if hasattr(self._strategy, 'browser_process'):
+            self.managed_browser = self._strategy
+        
+        # Set Playwright reference
+        self.playwright = self._strategy.playwright
+        
+        # Sync sessions if needed
+        if hasattr(self._strategy, 'sessions'):
+            self.sessions = self._strategy.sessions
+            self.session_ttl = self._strategy.session_ttl
         
         return self
-
-    async def _create_and_add_browser(self, browser_config: BrowserConfig, config_hash: str):
-        """Create and add a browser to the pool.
-        
-        Args:
-            browser_config: Browser configuration
-            config_hash: Hash of the configuration
-        """
-        try:
-            strategy = self._create_strategy(browser_config)
-            await strategy.start()
-            
-            async with self._browser_pool_lock:
-                if config_hash not in self.browser_pool:
-                    self.browser_pool[config_hash] = []
-                self.browser_pool[config_hash].append(strategy)
-                self.browser_in_use[strategy] = False
-            
-            if self.logger:
-                self.logger.debug(
-                    f"Added browser to pool: {browser_config.browser_type} "
-                    f"({browser_config.browser_mode})", 
-                    tag="POOL"
-                )
-        except Exception as e:
-            if self.logger:
-                self.logger.error(
-                    f"Failed to create browser: {str(e)}", 
-                    tag="POOL"
-                )
-            raise
-
-    def _make_config_signature(self, crawlerRunConfig: CrawlerRunConfig) -> str:
-        """Create a signature hash from crawler configuration.
-        
-        Args:
-            crawlerRunConfig: Crawler run configuration
-            
-        Returns:
-            str: Hash of the crawler configuration
-        """
-        config_dict = crawlerRunConfig.__dict__.copy()
-        # Exclude items that do not affect page creation
-        ephemeral_keys = [
-            "session_id",
-            "js_code",
-            "scraping_strategy",
-            "extraction_strategy",
-            "chunking_strategy",
-            "cache_mode",
-            "content_filter",
-            "semaphore_count",
-            "url"
-        ]
-        for key in ephemeral_keys:
-            if key in config_dict:
-                del config_dict[key]
-                
-        # Convert to canonical JSON string
-        config_json = json.dumps(config_dict, sort_keys=True, default=str)
-
-        # Hash the JSON
-        config_hash = hashlib.sha256(config_json.encode("utf-8")).hexdigest()
-        return config_hash            
-
-    async def _prewarm_pages(
-        self, 
-        browser_config: BrowserConfig, 
-        crawler_run_config: CrawlerRunConfig, 
-        count: int
-    ):
-        """Pre-warm pages for a specific configuration.
-        
-        Args:
-            browser_config: Browser configuration
-            crawler_run_config: Crawler run configuration
-            count: Number of pages to pre-warm
-        """
-        try:
-            # Create individual page tasks and run them in parallel
-            browser_config_hash = self._create_browser_config_hash(browser_config)
-            crawler_config_hash = self._make_config_signature(crawler_run_config)            
-            async def get_single_page():
-                strategy = await self.get_available_browser(browser_config)
-                try:
-                    page, context = await strategy.get_page(crawler_run_config)
-                    # Store config hashes on the page object for later retrieval
-                    setattr(page, "_browser_config_hash", browser_config_hash)
-                    setattr(page, "_crawler_config_hash", crawler_config_hash)                    
-                    return page, context, strategy
-                except Exception as e:
-                    # Release the browser back to the pool
-                    await self.release_browser(strategy, browser_config)
-                    raise e
-                
-            # Create tasks for parallel execution
-            page_tasks = [get_single_page() for _ in range(count)]
-            
-            # Execute all page creation tasks in parallel
-            pages_contexts_strategies = await asyncio.gather(*page_tasks)
-            
-            # Add pages to the page pool
-            browser_config_hash = self._create_browser_config_hash(browser_config)
-            crawler_config_hash = self._make_config_signature(crawler_run_config)
-            pool_key = (browser_config_hash, crawler_config_hash)
-            
-            async with self._page_pool_lock:
-                if pool_key not in self.page_pool:
-                    self.page_pool[pool_key] = []
-                
-                # Add all pages to the pool
-                self.page_pool[pool_key].extend(pages_contexts_strategies)
-                
-            if self.logger:
-                self.logger.debug(
-                    f"Pre-warmed {count} pages in parallel with config {crawler_run_config}", 
-                    tag="POOL"
-                )
-        except Exception as e:
-            if self.logger:
-                self.logger.error(
-                    f"Failed to pre-warm pages: {str(e)}", 
-                    tag="POOL"
-                )
-            raise
     
-    async def get_available_browser(
-        self, 
-        browser_config: Optional[BrowserConfig] = None
-    ) -> BaseBrowserStrategy:
-        """Get an available browser from the pool for the given configuration.
+    async def get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
+        """Get a page for the given configuration.
         
         Args:
-            browser_config: Browser configuration to match
+            crawlerRunConfig: Configuration object for the crawler run
             
         Returns:
-            BaseBrowserStrategy: An available browser strategy
-            
-        Raises:
-            Exception: If no browser is available and behavior is EXCEPTION
+            Tuple of (Page, BrowserContext)
         """
-        browser_config = browser_config or self.config
-        config_hash = self._create_browser_config_hash(browser_config)
+        # Delegate to strategy
+        page, context = await self._strategy.get_page(crawlerRunConfig)
         
-        async with self._browser_locks.get(config_hash, asyncio.Lock()):
-            # Check if we have browsers for this config
-            if config_hash not in self.browser_pool or not self.browser_pool[config_hash]:
-                if self.unavailable_behavior == UnavailableBehavior.ON_DEMAND:
-                    # Create a new browser on demand
-                    if self.logger:
-                        self.logger.info(
-                            f"1> Creating new browser on demand for config {config_hash[:8]}",
-                            tag="POOL"
-                        )
-                    
-                    # Initialize pool for this config if needed
-                    async with self._browser_pool_lock:
-                        if config_hash not in self.browser_pool:
-                            self.browser_pool[config_hash] = []
-                        
-                        strategy = self._create_strategy(browser_config)
-                        await strategy.start()
-
-                        self.browser_pool[config_hash].append(strategy)
-                        self.browser_in_use[strategy] = False
-
-                elif self.unavailable_behavior == UnavailableBehavior.EXCEPTION:
-                    raise Exception(f"No browsers available for configuration {config_hash[:8]}")
-            
-            # Check for an available browser with capacity in the pool
-            for strategy in self.browser_pool[config_hash]:
-                # Check if this browser has capacity for more pages
-                async with self._page_count_lock:
-                    current_pages = self.browser_page_counts.get(strategy, 0)
-
-                    if current_pages < self.max_pages_per_browser:
-                        # Increment the page count
-                        self.browser_page_counts[strategy] = current_pages + 1
-
-                        self.browser_in_use[strategy] = True
-
-                        # Get browser information for better logging
-                        browser_type = getattr(strategy.config, 'browser_type', 'unknown')
-                        browser_mode = getattr(strategy.config, 'browser_mode', 'unknown')
-                        strategy_id = id(strategy)  # Use object ID as a unique identifier
-
-                        if self.logger:
-                            self.logger.debug(
-                                f"Selected browser #{strategy_id} ({browser_type}/{browser_mode}) - " 
-                                f"pages: {current_pages+1}/{self.max_pages_per_browser}",
-                                tag="POOL"
-                            )                        
-
-                        return strategy
-            
-            # All browsers are at capacity or in use
-            if self.unavailable_behavior == UnavailableBehavior.ON_DEMAND:
-                # Check if we've reached the maximum number of browsers
-                if len(self.browser_pool[config_hash]) >= self.max_browsers_per_config:
-                    if self.logger:
-                        self.logger.warning(
-                            f"Maximum browsers reached for config {config_hash[:8]} and all at page capacity",
-                            tag="POOL"
-                        )
-                    if self.unavailable_behavior == UnavailableBehavior.EXCEPTION:
-                        raise Exception("Maximum browsers reached and all at page capacity")
-                
-                # Create a new browser on demand
-                if self.logger:
-                    self.logger.info(
-                        f"2> Creating new browser on demand for config {config_hash[:8]}",
-                        tag="POOL"
-                    )
-                
-                strategy = self._create_strategy(browser_config)
-                await strategy.start()
-                
-                async with self._browser_pool_lock:
-                    self.browser_pool[config_hash].append(strategy)
-                    self.browser_in_use[strategy] = True
-                
-                return strategy
-            
-            # If we get here, either behavior is EXCEPTION or PENDING
-            if self.unavailable_behavior == UnavailableBehavior.EXCEPTION:
-                raise Exception(f"All browsers in use or at page capacity for configuration {config_hash[:8]}")
-            
-            # For PENDING behavior, set up waiting mechanism
-            if config_hash not in self.request_queues:
-                self.request_queues[config_hash] = asyncio.Queue()
-            
-            # Create a future to wait on
-            future = asyncio.Future()
-            await self.request_queues[config_hash].put(future)
-            
-            if self.logger:
-                self.logger.debug(
-                    f"Waiting for available browser for config {config_hash[:8]}",
-                    tag="POOL"
-                )
-            
-            # Wait for a browser to become available
-            strategy = await future
-            return strategy
-
-    async def get_page(
-        self, 
-        crawlerRunConfig: CrawlerRunConfig,
-        browser_config: Optional[BrowserConfig] = None
-    ) -> Tuple[Page, BrowserContext, BaseBrowserStrategy]:
-        """Get a page from the browser pool."""
-        browser_config = browser_config or self.config
+        # Sync sessions if needed
+        if hasattr(self._strategy, 'sessions'):
+            self.sessions = self._strategy.sessions
         
-        # Check if we have a pre-warmed page available
-        browser_config_hash = self._create_browser_config_hash(browser_config)
-        crawler_config_hash = self._make_config_signature(crawlerRunConfig)
-        pool_key = (browser_config_hash, crawler_config_hash)
+        return page, context
         
-        # Try to get a page from the pool
-        async with self._page_pool_lock:
-            if pool_key in self.page_pool and self.page_pool[pool_key]:
-                # Get a page from the pool
-                page, context, strategy = self.page_pool[pool_key].pop()
-                
-                # Mark browser as in use (it already is, but ensure consistency)
-                self.browser_in_use[strategy] = True
-                
-                if self.logger:
-                    self.logger.debug(
-                        f"Using pre-warmed page for config {crawler_config_hash[:8]}", 
-                        tag="POOL"
-                    )
-                    
-                # Note: We don't increment page count since it was already counted when created
-                
-                return page, context, strategy
+    async def get_pages(self, crawlerRunConfig: CrawlerRunConfig, count: int = 1) -> List[Tuple[Page, BrowserContext]]:
+        """Get multiple pages with the same configuration.
         
-        # No pre-warmed page available, create a new one
-        # get_available_browser already increments the page count
-        strategy = await self.get_available_browser(browser_config)
+        This method efficiently creates multiple browser pages using the same configuration,
+        which is useful for parallel crawling of multiple URLs.
         
-        try:
-            # Get a page from the browser
-            page, context = await strategy.get_page(crawlerRunConfig)
+        Args:
+            crawlerRunConfig: Configuration for the pages
+            count: Number of pages to create
             
-            # Store config hashes on the page object for later retrieval
-            setattr(page, "_browser_config_hash", browser_config_hash)
-            setattr(page, "_crawler_config_hash", crawler_config_hash)
+        Returns:
+            List of (Page, Context) tuples
+        """
+        # Delegate to strategy
+        pages = await self._strategy.get_pages(crawlerRunConfig, count)
+        
+        # Sync sessions if needed
+        if hasattr(self._strategy, 'sessions'):
+            self.sessions = self._strategy.sessions
             
-            return page, context, strategy
-        except Exception as e:
-            # Release the browser back to the pool and decrement the page count
-            await self.release_browser(strategy, browser_config, decrement_page_count=True)
-            raise e
-        
-    async def release_page(
-        self, 
-        page: Page, 
-        strategy: BaseBrowserStrategy, 
-        browser_config: Optional[BrowserConfig] = None,
-        keep_alive: bool = True,
-        return_to_pool: bool = True
-    ):
-        """Release a page back to the pool."""
-        browser_config = browser_config or self.config
-
-        page_url = page.url if page else None
-        
-        # If not keeping the page alive, close it and decrement count
-        if not keep_alive:
-            try:
-                await page.close()
-            except Exception as e:
-                if self.logger:
-                    self.logger.error(
-                        f"Error closing page: {str(e)}",
-                        tag="POOL"
-                    )
-            # Release the browser with page count decrement
-            await self.release_browser(strategy, browser_config, decrement_page_count=True)
-            return
-        
-        # If returning to pool
-        if return_to_pool:
-            # Get the configuration hashes from the page object
-            browser_config_hash = getattr(page, "_browser_config_hash", None)
-            crawler_config_hash = getattr(page, "_crawler_config_hash", None)
-            
-            if browser_config_hash and crawler_config_hash:
-                pool_key = (browser_config_hash, crawler_config_hash)
-                
-                async with self._page_pool_lock:
-                    if pool_key not in self.page_pool:
-                        self.page_pool[pool_key] = []
-                    
-                    # Add page back to the pool
-                    self.page_pool[pool_key].append((page, page.context, strategy))
-                    
-                    if self.logger:
-                        self.logger.debug(
-                            f"Returned page to pool for config {crawler_config_hash[:8]}, url: {page_url}", 
-                            tag="POOL"
-                        )
-                    
-                    # Note: We don't decrement the page count here since the page is still "in use" 
-                    # from the browser's perspective, just in our pool
-                    return
-            else:
-                # If we can't identify the configuration, log a warning
-                if self.logger:
-                    self.logger.warning(
-                        "Cannot return page to pool - missing configuration hashes", 
-                        tag="POOL"
-                    )
-        
-        # If we got here, we couldn't return to pool, so just release the browser
-        await self.release_browser(strategy, browser_config, decrement_page_count=True)
+        return pages
     
-    async def release_browser(
-        self, 
-        strategy: BaseBrowserStrategy, 
-        browser_config: Optional[BrowserConfig] = None,
-        decrement_page_count: bool = True
-    ):
-        """Release a browser back to the pool."""
-        browser_config = browser_config or self.config
-        config_hash = self._create_browser_config_hash(browser_config)
-        
-        # Decrement page count
-        if decrement_page_count:
-            async with self._page_count_lock:
-                current_count = self.browser_page_counts.get(strategy, 1)
-                self.browser_page_counts[strategy] = max(0, current_count - 1)
-                
-                if self.logger:
-                    self.logger.debug(
-                        f"Decremented page count for browser (now: {self.browser_page_counts[strategy]})",
-                        tag="POOL"
-                    )
-        
-        # Mark as not in use
-        self.browser_in_use[strategy] = False
-        
-        # Process any waiting requests
-        if config_hash in self.request_queues and not self.request_queues[config_hash].empty():
-            future = await self.request_queues[config_hash].get()
-            if not future.done():
-                future.set_result(strategy)
-
-    async def get_pages(
-        self, 
-        crawlerRunConfig: CrawlerRunConfig, 
-        count: int = 1,
-        browser_config: Optional[BrowserConfig] = None
-    ) -> List[Tuple[Page, BrowserContext, BaseBrowserStrategy]]:
-        """Get multiple pages from the browser pool.
-        
-        Args:
-            crawlerRunConfig: Configuration for the crawler run
-            count: Number of pages to get
-            browser_config: Browser configuration to use
-            
-        Returns:
-            List of (Page, Context, Strategy) tuples
-        """
-        results = []
-        for _ in range(count):
-            try:
-                result = await self.get_page(crawlerRunConfig, browser_config)
-                results.append(result)
-            except Exception as e:
-                # Release any pages we've already gotten
-                for page, _, strategy in results:
-                    await self.release_page(page, strategy, browser_config)
-                raise e
-        
-        return results
-
-    async def get_page_pool_status(self) -> Dict[str, Any]:
-        """Get information about the page pool status.
-        
-        Returns:
-            Dict with page pool status information
-        """
-        status = {
-            "total_pooled_pages": 0,
-            "configs": {}
-        }
-        
-        async with self._page_pool_lock:
-            for (browser_hash, crawler_hash), pages in self.page_pool.items():
-                config_key = f"{browser_hash[:8]}_{crawler_hash[:8]}"
-                status["configs"][config_key] = len(pages)
-                status["total_pooled_pages"] += len(pages)
-        
-        if self.logger:
-            self.logger.debug(
-                f"Page pool status: {status['total_pooled_pages']} pages available",
-                tag="POOL"
-            )
-        
-        return status
-
-    async def get_pool_status(self) -> Dict[str, Any]:
-        """Get information about the browser pool status.
-        
-        Returns:
-            Dict with pool status information
-        """
-        status = {
-            "total_browsers": 0,
-            "browsers_in_use": 0,
-            "total_pages": 0,
-            "configs": {}
-        }
-        
-        for config_hash, strategies in self.browser_pool.items():
-            config_pages = 0
-            in_use = 0
-            
-            for strategy in strategies:
-                is_in_use = self.browser_in_use.get(strategy, False)
-                if is_in_use:
-                    in_use += 1
-                
-                # Get page count for this browser
-                try:
-                    page_count = len(await strategy.get_opened_pages())
-                    config_pages += page_count
-                except Exception as e:
-                    if self.logger:
-                        self.logger.error(f"Error getting page count: {str(e)}", tag="POOL")
-            
-            config_status = {
-                "total_browsers": len(strategies),
-                "browsers_in_use": in_use,
-                "pages_open": config_pages,
-                "waiting_requests": self.request_queues.get(config_hash, asyncio.Queue()).qsize(),
-                "max_capacity": len(strategies) * self.max_pages_per_browser,
-                "utilization_pct": round((config_pages / (len(strategies) * self.max_pages_per_browser)) * 100, 1) 
-                    if strategies else 0
-            }
-            
-            status["configs"][config_hash] = config_status
-            status["total_browsers"] += config_status["total_browsers"]
-            status["browsers_in_use"] += config_status["browsers_in_use"]
-            status["total_pages"] += config_pages
-        
-        # Add overall utilization
-        if status["total_browsers"] > 0:
-            max_capacity = status["total_browsers"] * self.max_pages_per_browser
-            status["overall_utilization_pct"] = round((status["total_pages"] / max_capacity) * 100, 1)
-        else:
-            status["overall_utilization_pct"] = 0
-        
-        return status
-
-    async def start(self):
-        """Start at least one browser instance in the pool.
-        
-        This method is kept for backward compatibility.
-        
-        Returns:
-            self: For method chaining
-        """
-        await self.initialize_pool([self.config], 1)
-        return self
-        
     async def kill_session(self, session_id: str):
         """Kill a browser session and clean up resources.
         
-        Delegated to the strategy. This method is kept for backward compatibility.
-        
         Args:
             session_id: The session ID to kill
         """
-        if not self.strategy:
+        # Handle kill_session via our strategy if it supports it
+        if hasattr(self._strategy, '_kill_session'):
+            await self._strategy._kill_session(session_id)
+        elif session_id in self.sessions:
+            context, page, _ = self.sessions[session_id]
+            await page.close()
+            # Only close context if not using CDP
+            if not self.config.use_managed_browser and not self.config.cdp_url and not self.config.browser_mode == "builtin":
+                await context.close()
+            del self.sessions[session_id]
+    
+    def _cleanup_expired_sessions(self):
+        """Clean up expired sessions based on TTL."""
+        # Use strategy's implementation if available
+        if hasattr(self._strategy, '_cleanup_expired_sessions'):
+            self._strategy._cleanup_expired_sessions()
             return
-            
-        await self.strategy.kill_session(session_id)
         
-        # Sync sessions
-        if hasattr(self.strategy, 'sessions'):
-            self.sessions = self.strategy.sessions
+        # Otherwise use our own implementation
+        current_time = time.time()
+        expired_sessions = [
+            sid
+            for sid, (_, _, last_used) in self.sessions.items()
+            if current_time - last_used > self.session_ttl
+        ]
+        for sid in expired_sessions:
+            asyncio.create_task(self.kill_session(sid))
     
     async def close(self):
-        """Close all browsers in the pool and clean up resources."""
-        # Close all browsers in the pool
-        for strategies in self.browser_pool.values():
-            for strategy in strategies:
-                try:
-                    await strategy.close()
-                except Exception as e:
-                    if self.logger:
-                        self.logger.error(
-                            f"Error closing browser: {str(e)}",
-                            tag="POOL"
-                        )
-        
-        # Clear pool data
-        self.browser_pool = {}
-        self.browser_in_use = {}
+        """Close the browser and clean up resources."""
+        # Delegate to strategy
+        await self._strategy.close()
         
         # Reset legacy references
         self.browser = None
         self.default_context = None
         self.managed_browser = None
         self.playwright = None
-        self.strategy = None
         self.sessions = {}
-
-
-async def create_browser_manager(
-    browser_config: Optional[BrowserConfig] = None,
-    logger: Optional[AsyncLogger] = None,
-    unavailable_behavior: UnavailableBehavior = UnavailableBehavior.EXCEPTION,
-    max_browsers_per_config: int = 10,
-    initial_pool_size: int = 1,
-    page_configs: Optional[List[Tuple[BrowserConfig, CrawlerRunConfig, int]]] = None
-) -> BrowserManager:
-    """Factory function to create and initialize a BrowserManager.
-    
-    Args:
-        browser_config: Configuration for the browsers
-        logger: Logger for recording events
-        unavailable_behavior: Behavior when no browser is available
-        max_browsers_per_config: Maximum browsers per configuration
-        initial_pool_size: Initial number of browsers per configuration
-        page_configs: Optional configurations for pre-warming pages
-        
-    Returns:
-        Initialized BrowserManager
-    """
-    manager = BrowserManager(
-        browser_config=browser_config,
-        logger=logger,
-        unavailable_behavior=unavailable_behavior,
-        max_browsers_per_config=max_browsers_per_config
-    )
-    
-    await manager.initialize_pool(
-        [browser_config] if browser_config else None,
-        initial_pool_size,
-        page_configs
-    )
-    
-    return manager
-
-
-
-
-

crawl4ai/browser/models.py

@@ -1,143 +0,0 @@
-"""Docker configuration module for Crawl4AI browser automation.
-
-This module provides configuration classes for Docker-based browser automation,
-allowing flexible configuration of Docker containers for browsing.
-"""
-
-from typing import Dict, List, Optional
-
-
-class DockerConfig:
-    """Configuration for Docker-based browser automation.
-    
-    This class contains Docker-specific settings to avoid cluttering BrowserConfig.
-    
-    Attributes:
-        mode (str): Docker operation mode - "connect" or "launch".
-            - "connect": Uses a container with Chrome already running
-            - "launch": Dynamically configures and starts Chrome in container
-        image (str): Docker image to use. If None, defaults from DockerUtils are used.
-        registry_file (str): Path to container registry file for persistence.
-        persistent (bool): Keep container running after browser closes.
-        remove_on_exit (bool): Remove container on exit when not persistent.
-        network (str): Docker network to use.
-        volumes (List[str]): Volume mappings (e.g., ["host_path:container_path"]).
-        env_vars (Dict[str, str]): Environment variables to set in container.
-        extra_args (List[str]): Additional docker run arguments.
-        host_port (int): Host port to map to container's 9223 port.
-        user_data_dir (str): Path to user data directory on host.
-        container_user_data_dir (str): Path to user data directory in container.
-    """
-    
-    def __init__(
-        self,
-        mode: str = "connect",                     # "connect" or "launch" 
-        image: Optional[str] = None,               # Docker image to use
-        registry_file: Optional[str] = None,       # Path to registry file
-        persistent: bool = False,                  # Keep container running after browser closes
-        remove_on_exit: bool = True,               # Remove container on exit when not persistent
-        network: Optional[str] = None,             # Docker network to use
-        volumes: List[str] = None,                 # Volume mappings
-        cpu_limit: float = 1.0,                    # CPU limit for the container
-        memory_limit: str = "1.5g",                # Memory limit for the container
-        env_vars: Dict[str, str] = None,           # Environment variables
-        host_port: Optional[int] = None,           # Host port to map to container's 9223
-        user_data_dir: Optional[str] = None,       # Path to user data directory on host
-        container_user_data_dir: str = "/data",    # Path to user data directory in container
-        extra_args: List[str] = None,              # Additional docker run arguments
-    ):
-        """Initialize Docker configuration.
-        
-        Args:
-            mode: Docker operation mode ("connect" or "launch")
-            image: Docker image to use
-            registry_file: Path to container registry file
-            persistent: Whether to keep container running after browser closes
-            remove_on_exit: Whether to remove container on exit when not persistent
-            network: Docker network to use
-            volumes: Volume mappings as list of strings
-            cpu_limit: CPU limit for the container
-            memory_limit: Memory limit for the container
-            env_vars: Environment variables as dictionary
-            extra_args: Additional docker run arguments
-            host_port: Host port to map to container's 9223
-            user_data_dir: Path to user data directory on host
-            container_user_data_dir: Path to user data directory in container
-        """
-        self.mode = mode
-        self.image = image  # If None, defaults will be used from DockerUtils
-        self.registry_file = registry_file
-        self.persistent = persistent
-        self.remove_on_exit = remove_on_exit
-        self.network = network
-        self.volumes = volumes or []
-        self.cpu_limit = cpu_limit
-        self.memory_limit = memory_limit
-        self.env_vars = env_vars or {}
-        self.extra_args = extra_args or []
-        self.host_port = host_port
-        self.user_data_dir = user_data_dir
-        self.container_user_data_dir = container_user_data_dir
-    
-    def to_dict(self) -> Dict:
-        """Convert this configuration to a dictionary.
-        
-        Returns:
-            Dictionary representation of this configuration
-        """
-        return {
-            "mode": self.mode,
-            "image": self.image,
-            "registry_file": self.registry_file,
-            "persistent": self.persistent,
-            "remove_on_exit": self.remove_on_exit,
-            "network": self.network,
-            "volumes": self.volumes,
-            "cpu_limit": self.cpu_limit,
-            "memory_limit": self.memory_limit,
-            "env_vars": self.env_vars,
-            "extra_args": self.extra_args,
-            "host_port": self.host_port,
-            "user_data_dir": self.user_data_dir,
-            "container_user_data_dir": self.container_user_data_dir
-        }
-        
-    @staticmethod
-    def from_kwargs(kwargs: Dict) -> "DockerConfig":
-        """Create a DockerConfig from a dictionary of keyword arguments.
-        
-        Args:
-            kwargs: Dictionary of configuration options
-            
-        Returns:
-            New DockerConfig instance
-        """
-        return DockerConfig(
-            mode=kwargs.get("mode", "connect"),
-            image=kwargs.get("image"),
-            registry_file=kwargs.get("registry_file"),
-            persistent=kwargs.get("persistent", False),
-            remove_on_exit=kwargs.get("remove_on_exit", True),
-            network=kwargs.get("network"),
-            volumes=kwargs.get("volumes"),
-            cpu_limit=kwargs.get("cpu_limit", 1.0),
-            memory_limit=kwargs.get("memory_limit", "1.5g"),
-            env_vars=kwargs.get("env_vars"),
-            extra_args=kwargs.get("extra_args"),
-            host_port=kwargs.get("host_port"),
-            user_data_dir=kwargs.get("user_data_dir"),
-            container_user_data_dir=kwargs.get("container_user_data_dir", "/data")
-        )
-        
-    def clone(self, **kwargs) -> "DockerConfig":
-        """Create a copy of this configuration with updated values.
-        
-        Args:
-            **kwargs: Key-value pairs of configuration options to update
-            
-        Returns:
-            DockerConfig: A new instance with the specified updates
-        """
-        config_dict = self.to_dict()
-        config_dict.update(kwargs)
-        return DockerConfig.from_kwargs(config_dict)

crawl4ai/browser/strategies/__init__.py

@@ -1,13 +0,0 @@
-from .base import BaseBrowserStrategy
-from .cdp import CDPBrowserStrategy
-from .docker_strategy import DockerBrowserStrategy
-from .playwright import PlaywrightBrowserStrategy
-from .builtin import BuiltinBrowserStrategy
-
-__all__ = [
-    "BrowserStrategy",
-    "CDPBrowserStrategy",
-    "DockerBrowserStrategy",
-    "PlaywrightBrowserStrategy",
-    "BuiltinBrowserStrategy",
-]

crawl4ai/browser/strategies/base.py

@@ -1,601 +0,0 @@
-"""Browser strategies module for Crawl4AI.
-
-This module implements the browser strategy pattern for different
-browser implementations, including Playwright, CDP, and builtin browsers.
-"""
-
-from abc import ABC, abstractmethod
-import asyncio
-import json
-import hashlib
-import os
-import time
-from typing import Optional, Tuple, List
-
-from playwright.async_api import BrowserContext, Page
-
-from ...async_logger import AsyncLogger
-from ...async_configs import BrowserConfig, CrawlerRunConfig
-from ...config import DOWNLOAD_PAGE_TIMEOUT
-from ...js_snippet import load_js_script
-from ..utils import get_playwright
-
-
-class BaseBrowserStrategy(ABC):
-    """Base class for all browser strategies.
-    
-    This abstract class defines the interface that all browser strategies
-    must implement. It handles common functionality like context caching,
-    browser configuration, and session management.
-    """
-    
-    _playwright_instance = None
-    
-    @classmethod
-    async def get_playwright(cls):
-        """Get or create a shared Playwright instance.
-        
-        Returns:
-            Playwright: The shared Playwright instance
-        """
-        # For now I dont want Singleton pattern for Playwright
-        if cls._playwright_instance is None or True:
-            cls._playwright_instance = await get_playwright()
-        return cls._playwright_instance
-        
-    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
-        """Initialize the strategy with configuration and logger.
-        
-        Args:
-            config: Browser configuration
-            logger: Logger for recording events and errors
-        """
-        self.config = config
-        self.logger = logger
-        self.browser = None
-        self.default_context = None
-        
-        # Context management
-        self.contexts_by_config = {}  # config_signature -> context
-
-        self._contexts_lock = asyncio.Lock()
-        
-        # Session management
-        self.sessions = {}
-        self.session_ttl = 1800  # 30 minutes default
-        
-        # Playwright instance
-        self.playwright = None
-    
-    @abstractmethod
-    async def start(self):
-        """Start the browser.
-        
-        This method should be implemented by concrete strategies to initialize 
-        the browser in the appropriate way (direct launch, CDP connection, etc.)
-        
-        Returns:
-            self: For method chaining
-        """
-        # Base implementation gets the playwright instance
-        self.playwright = await self.get_playwright()
-        return self
-    
-    @abstractmethod
-    async def _generate_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
-        pass
-
-    async def get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
-        """Get a page with specified configuration.
-        
-        This method should be implemented by concrete strategies to create
-        or retrieve a page according to their browser management approach.
-        
-        Args:
-            crawlerRunConfig: Crawler run configuration
-            
-        Returns:
-            Tuple of (Page, BrowserContext)
-        """
-        # Clean up expired sessions first
-        self._cleanup_expired_sessions()
-
-        # If a session_id is provided and we already have it, reuse that page + context
-        if crawlerRunConfig.session_id and crawlerRunConfig.session_id in self.sessions:
-            context, page, _ = self.sessions[crawlerRunConfig.session_id]
-            # Update last-used timestamp
-            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
-            return page, context
-        
-        page, context = await self._generate_page(crawlerRunConfig)
-
-        import uuid
-        setattr(page, "guid", uuid.uuid4())
-
-        # If a session_id is specified, store this session so we can reuse later
-        if crawlerRunConfig.session_id:
-            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
-            
-        return page, context        
-        pass
-    
-    async def get_pages(self, crawlerRunConfig: CrawlerRunConfig, count: int = 1) -> List[Tuple[Page, BrowserContext]]:
-        """Get multiple pages with the same configuration.
-        
-        Args:
-            crawlerRunConfig: Configuration for the pages
-            count: Number of pages to create
-            
-        Returns:
-            List of (Page, Context) tuples
-        """
-        pages = []
-        for _ in range(count):
-            page, context = await self.get_page(crawlerRunConfig)
-            pages.append((page, context))
-        return pages
-       
-    async def get_opened_pages(self) -> List[Page]:
-        """Get all opened pages in the
-        browser.
-        """
-        return [page for context in self.contexts_by_config.values() for page in context.pages]
-
-    def _build_browser_args(self) -> dict:
-        """Build browser launch arguments from config.
-        
-        Returns:
-            dict: Browser launch arguments for Playwright
-        """
-        # Define common browser arguments that improve performance and stability
-        args = [
-            "--no-sandbox",
-            "--no-first-run",
-            "--no-default-browser-check",
-            "--window-position=0,0",
-            "--ignore-certificate-errors",
-            "--ignore-certificate-errors-spki-list",
-            "--window-position=400,0",
-            "--force-color-profile=srgb",
-            "--mute-audio",
-            "--disable-gpu",
-            "--disable-gpu-compositing",
-            "--disable-software-rasterizer",
-            "--disable-dev-shm-usage",
-            "--disable-infobars",
-            "--disable-blink-features=AutomationControlled",
-            "--disable-renderer-backgrounding",
-            "--disable-ipc-flooding-protection",
-            "--disable-background-timer-throttling",
-            f"--window-size={self.config.viewport_width},{self.config.viewport_height}",
-        ]
-
-        # Define browser disable options for light mode
-        browser_disable_options = [
-            "--disable-backgrounding-occluded-windows",
-            "--disable-breakpad",
-            "--disable-client-side-phishing-detection",
-            "--disable-component-extensions-with-background-pages",
-            "--disable-default-apps",
-            "--disable-extensions",
-            "--disable-features=TranslateUI",
-            "--disable-hang-monitor",
-            "--disable-popup-blocking",
-            "--disable-prompt-on-repost",
-            "--disable-sync",
-            "--metrics-recording-only",
-            "--password-store=basic",
-            "--use-mock-keychain",
-        ]
-
-        # Apply light mode settings if enabled
-        if self.config.light_mode:
-            args.extend(browser_disable_options)
-
-        # Apply text mode settings if enabled (disables images, JS, etc)
-        if self.config.text_mode:
-            args.extend([
-                "--blink-settings=imagesEnabled=false",
-                "--disable-remote-fonts",
-                "--disable-images",
-                "--disable-javascript",
-                "--disable-software-rasterizer",
-                "--disable-dev-shm-usage",
-            ])
-
-        # Add any extra arguments from the config
-        if self.config.extra_args:
-            args.extend(self.config.extra_args)
-
-        # Build the core browser args dictionary
-        browser_args = {"headless": self.config.headless, "args": args}
-
-        # Add chrome channel if specified
-        if self.config.chrome_channel:
-            browser_args["channel"] = self.config.chrome_channel
-
-        # Configure downloads
-        if self.config.accept_downloads:
-            browser_args["downloads_path"] = self.config.downloads_path or os.path.join(
-                os.getcwd(), "downloads"
-            )
-            os.makedirs(browser_args["downloads_path"], exist_ok=True)
-
-        # Check for user data directory
-        if self.config.user_data_dir:
-            # Ensure the directory exists
-            os.makedirs(self.config.user_data_dir, exist_ok=True)
-            browser_args["user_data_dir"] = self.config.user_data_dir
-        
-        # Configure proxy settings
-        if self.config.proxy or self.config.proxy_config:
-            from playwright.async_api import ProxySettings
-
-            proxy_settings = (
-                ProxySettings(server=self.config.proxy)
-                if self.config.proxy
-                else ProxySettings(
-                    server=self.config.proxy_config.server,
-                    username=self.config.proxy_config.username,
-                    password=self.config.proxy_config.password,
-                )
-            )
-            browser_args["proxy"] = proxy_settings
-
-        return browser_args
-    
-    def _make_config_signature(self, crawlerRunConfig: CrawlerRunConfig) -> str:
-        """Create a signature hash from configuration for context caching.
-        
-        Converts the crawlerRunConfig into a dict, excludes ephemeral fields,
-        then returns a hash of the sorted JSON. This yields a stable signature
-        that identifies configurations requiring a unique browser context.
-        
-        Args:
-            crawlerRunConfig: Crawler run configuration
-            
-        Returns:
-            str: Unique hash for this configuration
-        """
-        config_dict = crawlerRunConfig.__dict__.copy()
-        # Exclude items that do not affect browser-level setup
-        ephemeral_keys = [
-            "session_id",
-            "js_code",
-            "scraping_strategy",
-            "extraction_strategy",
-            "chunking_strategy",
-            "cache_mode",
-            "content_filter",
-            "semaphore_count",
-            "url"
-        ]
-        for key in ephemeral_keys:
-            if key in config_dict:
-                del config_dict[key]
-                
-        # Convert to canonical JSON string
-        signature_json = json.dumps(config_dict, sort_keys=True, default=str)
-
-        # Hash the JSON so we get a compact, unique string
-        signature_hash = hashlib.sha256(signature_json.encode("utf-8")).hexdigest()
-        return signature_hash
-        
-    async def create_browser_context(self, crawlerRunConfig: Optional[CrawlerRunConfig] = None) -> BrowserContext:
-        """Creates and returns a new browser context with configured settings.
-        
-        Args:
-            crawlerRunConfig: Configuration object for the crawler run
-            
-        Returns:
-            BrowserContext: Browser context object with the specified configurations
-        """
-        if not self.browser:
-            raise ValueError("Browser must be initialized before creating context")
-            
-        # Base settings
-        user_agent = self.config.headers.get("User-Agent", self.config.user_agent) 
-        viewport_settings = {
-            "width": self.config.viewport_width,
-            "height": self.config.viewport_height,
-        }
-        proxy_settings = {"server": self.config.proxy} if self.config.proxy else None
-        
-        # Define blocked extensions for resource optimization
-        blocked_extensions = [
-            # Images
-            "jpg", "jpeg", "png", "gif", "webp", "svg", "ico", "bmp", "tiff", "psd",
-            # Fonts
-            "woff", "woff2", "ttf", "otf", "eot",
-            # Media
-            "mp4", "webm", "ogg", "avi", "mov", "wmv", "flv", "m4v", "mp3", "wav", "aac",
-            "m4a", "opus", "flac",
-            # Documents
-            "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx",
-            # Archives
-            "zip", "rar", "7z", "tar", "gz",
-            # Scripts and data
-            "xml", "swf", "wasm",
-        ]
-
-        # Common context settings
-        context_settings = {
-            "user_agent": user_agent,
-            "viewport": viewport_settings,
-            "proxy": proxy_settings,
-            "accept_downloads": self.config.accept_downloads,
-            "storage_state": self.config.storage_state,
-            "ignore_https_errors": self.config.ignore_https_errors,
-            "device_scale_factor": 1.0,
-            "java_script_enabled": self.config.java_script_enabled,
-        }
-        
-        # Apply text mode settings if enabled
-        if self.config.text_mode:
-            text_mode_settings = {
-                "has_touch": False,
-                "is_mobile": False,
-                "java_script_enabled": False,  # Disable javascript in text mode
-            }
-            # Update context settings with text mode settings
-            context_settings.update(text_mode_settings)
-            if self.logger:
-                self.logger.debug("Text mode enabled for browser context", tag="BROWSER")
-        
-        # Handle storage state properly - this is key for persistence
-        if self.config.storage_state:
-            if self.logger:
-                if isinstance(self.config.storage_state, str):
-                    self.logger.debug(f"Using storage state from file: {self.config.storage_state}", tag="BROWSER")
-                else:
-                    self.logger.debug("Using storage state from config object", tag="BROWSER")
-
-        if self.config.user_data_dir:
-            # For CDP-based browsers, storage persistence is typically handled by the user_data_dir
-            # at the browser level, but we'll create a storage_state location for Playwright as well
-            storage_path = os.path.join(self.config.user_data_dir, "storage_state.json")
-            if not os.path.exists(storage_path):
-                # Create parent directory if it doesn't exist
-                os.makedirs(os.path.dirname(storage_path), exist_ok=True)
-                with open(storage_path, "w") as f:
-                    json.dump({}, f)
-            self.config.storage_state = storage_path
-
-            if self.logger:
-                self.logger.debug(f"Using user data directory: {self.config.user_data_dir}", tag="BROWSER")
-        
-        # Apply crawler-specific configurations if provided
-        if crawlerRunConfig:
-            # Check if there is value for crawlerRunConfig.proxy_config set add that to context
-            if crawlerRunConfig.proxy_config:
-                proxy_settings = {
-                    "server": crawlerRunConfig.proxy_config.server,
-                }
-                if crawlerRunConfig.proxy_config.username:
-                    proxy_settings.update({
-                        "username": crawlerRunConfig.proxy_config.username,
-                        "password": crawlerRunConfig.proxy_config.password,
-                    })
-                context_settings["proxy"] = proxy_settings
-                
-        # Create and return the context
-        try:
-            # Create the context with appropriate settings
-            context = await self.browser.new_context(**context_settings)
-            
-            # Apply text mode resource blocking if enabled
-            if self.config.text_mode:
-                # Create and apply route patterns for each extension
-                for ext in blocked_extensions:
-                    await context.route(f"**/*.{ext}", lambda route: route.abort())
-                    
-            return context
-        except Exception as e:
-            if self.logger:
-                self.logger.error(f"Error creating browser context: {str(e)}", tag="BROWSER")
-            # Fallback to basic context creation if the advanced settings fail
-            return await self.browser.new_context()
-        
-    async def setup_context(self, context: BrowserContext, crawlerRunConfig: Optional[CrawlerRunConfig] = None):
-        """Set up a browser context with the configured options.
-        
-        Args:
-            context: The browser context to set up
-            crawlerRunConfig: Configuration object containing all browser settings
-        """
-        # Set HTTP headers
-        if self.config.headers:
-            await context.set_extra_http_headers(self.config.headers)
-
-        # Add cookies
-        if self.config.cookies:
-            await context.add_cookies(self.config.cookies)
-
-        # Apply storage state if provided
-        if self.config.storage_state:
-            await context.storage_state(path=None)
-
-        # Configure downloads
-        if self.config.accept_downloads:
-            context.set_default_timeout(DOWNLOAD_PAGE_TIMEOUT)
-            context.set_default_navigation_timeout(DOWNLOAD_PAGE_TIMEOUT)
-            if self.config.downloads_path:
-                context._impl_obj._options["accept_downloads"] = True
-                context._impl_obj._options["downloads_path"] = self.config.downloads_path
-
-        # Handle user agent and browser hints
-        if self.config.user_agent:
-            combined_headers = {
-                "User-Agent": self.config.user_agent,
-                "sec-ch-ua": self.config.browser_hint,
-            }
-            combined_headers.update(self.config.headers)
-            await context.set_extra_http_headers(combined_headers)
-
-        # Add default cookie
-        target_url = (crawlerRunConfig and crawlerRunConfig.url) or "https://crawl4ai.com/"
-        await context.add_cookies(
-            [
-                {
-                    "name": "cookiesEnabled",
-                    "value": "true",
-                    "url": target_url,
-                }
-            ]
-        )
-
-        # Handle navigator overrides
-        if crawlerRunConfig:
-            if (
-                crawlerRunConfig.override_navigator
-                or crawlerRunConfig.simulate_user
-                or crawlerRunConfig.magic
-            ):
-                await context.add_init_script(load_js_script("navigator_overrider"))
-                
-    async def kill_session(self, session_id: str):
-        """Kill a browser session and clean up resources.
-
-        Args:
-            session_id (str): The session ID to kill.
-        """
-        if session_id not in self.sessions:
-            return
-            
-        context, page, _ = self.sessions[session_id]
-        
-        # Close the page
-        try:
-            await page.close()
-        except Exception as e:
-            if self.logger:
-                self.logger.error(f"Error closing page for session {session_id}: {str(e)}", tag="BROWSER")
-        
-        # Remove session from tracking
-        del self.sessions[session_id]
-        
-        # Clean up any contexts that no longer have pages
-        await self._cleanup_unused_contexts()
-        
-        if self.logger:
-            self.logger.debug(f"Killed session: {session_id}", tag="BROWSER")
-
-    async def _cleanup_unused_contexts(self):
-        """Clean up contexts that no longer have any pages."""
-        async with self._contexts_lock:
-            # Get all contexts we're managing
-            contexts_to_check = list(self.contexts_by_config.values())
-            
-            for context in contexts_to_check:
-                # Check if the context has any pages left
-                if not context.pages:
-                    # No pages left, we can close this context
-                    config_signature = next((sig for sig, ctx in self.contexts_by_config.items() 
-                                           if ctx == context), None)
-                    if config_signature:
-                        try:
-                            await context.close()
-                            del self.contexts_by_config[config_signature]
-                            if self.logger:
-                                self.logger.debug(f"Closed unused context", tag="BROWSER")
-                        except Exception as e:
-                            if self.logger:
-                                self.logger.error(f"Error closing unused context: {str(e)}", tag="BROWSER")
-    
-    def _cleanup_expired_sessions(self):
-        """Clean up expired sessions based on TTL."""
-        current_time = time.time()
-        expired_sessions = [
-            sid
-            for sid, (_, _, last_used) in self.sessions.items()
-            if current_time - last_used > self.session_ttl
-        ]
-        
-        for sid in expired_sessions:
-            if self.logger:
-                self.logger.debug(f"Session expired: {sid}", tag="BROWSER")
-            asyncio.create_task(self.kill_session(sid))
-
-    async def close(self):
-        """Close the browser and clean up resources.
-        
-        This method handles common cleanup tasks like:
-        1. Persisting storage state if a user_data_dir is configured
-        2. Closing all sessions
-        3. Closing all browser contexts
-        4. Closing the browser
-        5. Stopping Playwright
-        
-        Child classes should override this method to add their specific cleanup logic,
-        but should call super().close() to ensure common cleanup tasks are performed.
-        """
-        # Set a flag to prevent race conditions during cleanup
-        self.shutting_down = True
-        
-        try:
-            # Add brief delay if configured
-            if self.config.sleep_on_close:
-                await asyncio.sleep(0.5)
-                
-            # Persist storage state if using a user data directory
-            if self.config.user_data_dir and self.browser:
-                for context in self.browser.contexts:
-                    try:
-                        # Ensure the directory exists
-                        storage_dir = os.path.join(self.config.user_data_dir, "Default")
-                        os.makedirs(storage_dir, exist_ok=True)
-                        
-                        # Save storage state
-                        storage_path = os.path.join(storage_dir, "storage_state.json")
-                        await context.storage_state(path=storage_path)
-                        
-                        if self.logger:
-                            self.logger.debug("Storage state persisted before closing browser", tag="BROWSER")
-                    except Exception as e:
-                        if self.logger:
-                            self.logger.warning(
-                                message="Failed to ensure storage persistence: {error}",
-                                tag="BROWSER", 
-                                params={"error": str(e)}
-                            )
-            
-            # Close all active sessions
-            session_ids = list(self.sessions.keys())
-            for session_id in session_ids:
-                await self.kill_session(session_id)
-                
-            # Close all cached contexts
-            for ctx in self.contexts_by_config.values():
-                try:
-                    await ctx.close()
-                except Exception as e:
-                    if self.logger:
-                        self.logger.error(
-                            message="Error closing context: {error}",
-                            tag="BROWSER",
-                            params={"error": str(e)}
-                        )
-            self.contexts_by_config.clear()
-            
-            # Close the browser if it exists
-            if self.browser:
-                await self.browser.close()
-                self.browser = None
-                
-            # Stop playwright
-            if self.playwright:
-                await self.playwright.stop()
-                self.playwright = None
-                
-        except Exception as e:
-            if self.logger:
-                self.logger.error(
-                    message="Error during browser cleanup: {error}",
-                    tag="BROWSER",
-                    params={"error": str(e)}
-                )
-        finally:
-            # Reset shutting down flag
-            self.shutting_down = False
-    
-    

crawl4ai/browser/strategies/builtin.py

@@ -1,468 +0,0 @@
-import asyncio
-import os
-import time
-import json
-import subprocess
-import shutil
-import signal
-from typing import Optional, Dict, Any, Tuple
-
-
-from ...async_logger import AsyncLogger
-from ...async_configs import CrawlerRunConfig
-from playwright.async_api import Page, BrowserContext
-from ...async_logger import AsyncLogger
-from ...async_configs import BrowserConfig
-from ...utils import get_home_folder
-from ..utils import get_browser_executable, is_windows, is_browser_running, find_process_by_port, terminate_process
-
-
-from .cdp import CDPBrowserStrategy
-from .base import BaseBrowserStrategy
-
-class BuiltinBrowserStrategy(CDPBrowserStrategy):
-    """Built-in browser strategy.
-    
-    This strategy extends the CDP strategy to use the built-in browser.
-    """
-    
-    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
-        """Initialize the built-in browser strategy.
-        
-        Args:
-            config: Browser configuration
-            logger: Logger for recording events and errors
-        """
-        super().__init__(config, logger)
-        self.builtin_browser_dir = os.path.join(get_home_folder(), "builtin-browser") if not self.config.user_data_dir else self.config.user_data_dir
-        self.builtin_config_file = os.path.join(self.builtin_browser_dir, "browser_config.json")
-
-        # Raise error if user data dir is already engaged
-        if self._check_user_dir_is_engaged(self.builtin_browser_dir):
-            raise Exception(f"User data directory {self.builtin_browser_dir} is already engaged by another browser instance.")
-
-        os.makedirs(self.builtin_browser_dir, exist_ok=True)
-    
-    def _check_user_dir_is_engaged(self, user_data_dir: str) -> bool:
-        """Check if the user data directory is already in use.
-        
-        Returns:
-            bool: True if the directory is engaged, False otherwise
-        """
-        # Load browser config file, then iterate in port_map values, check "user_data_dir" key if it matches
-        # the current user data directory
-        if os.path.exists(self.builtin_config_file):
-            try:
-                with open(self.builtin_config_file, 'r') as f:
-                    browser_info_dict = json.load(f)
-                
-                # Check if user data dir is already engaged
-                for port_str, browser_info in browser_info_dict.get("port_map", {}).items():
-                    if browser_info.get("user_data_dir") == user_data_dir:
-                        return True
-            except Exception as e:
-                if self.logger:
-                    self.logger.error(f"Error reading built-in browser config: {str(e)}", tag="BUILTIN")
-        return False
-
-    async def start(self):
-        """Start or connect to the built-in browser.
-        
-        Returns:
-            self: For method chaining
-        """
-        # Initialize Playwright instance via base class method
-        await BaseBrowserStrategy.start(self)
-        
-        try:
-            # Check for existing built-in browser (get_browser_info already checks if running)
-            browser_info = self.get_browser_info()
-            if browser_info:
-                if self.logger:
-                    self.logger.info(f"Using existing built-in browser at {browser_info.get('cdp_url')}", tag="BROWSER")
-                self.config.cdp_url = browser_info.get('cdp_url')
-            else:
-                if self.logger:
-                    self.logger.info("Built-in browser not found, launching new instance...", tag="BROWSER")
-                cdp_url = await self.launch_builtin_browser(
-                    browser_type=self.config.browser_type,
-                    debugging_port=self.config.debugging_port,
-                    headless=self.config.headless,
-                )
-                if not cdp_url:
-                    if self.logger:
-                        self.logger.warning("Failed to launch built-in browser, falling back to regular CDP strategy", tag="BROWSER")
-                    # Call CDP's start but skip BaseBrowserStrategy.start() since we already called it
-                    return await CDPBrowserStrategy.start(self)
-                self.config.cdp_url = cdp_url
-            
-            # Connect to the browser using CDP protocol
-            self.browser = await self.playwright.chromium.connect_over_cdp(self.config.cdp_url)
-            
-            # Get or create default context
-            contexts = self.browser.contexts
-            if contexts:
-                self.default_context = contexts[0]
-            else:
-                self.default_context = await self.create_browser_context()
-            
-            await self.setup_context(self.default_context)
-            
-            if self.logger:
-                self.logger.debug(f"Connected to built-in browser at {self.config.cdp_url}", tag="BUILTIN")
-                
-            return self
-        except Exception as e:
-            if self.logger:
-                self.logger.error(f"Failed to start built-in browser: {str(e)}", tag="BUILTIN")
-            
-            # There is a possibility that at this point I need to clean up some resourece
-            raise
-
-    def _get_builtin_browser_info(cls, debugging_port: int, config_file: str, logger: Optional[AsyncLogger] = None) -> Optional[Dict[str, Any]]:
-        """Get information about the built-in browser for a specific debugging port.
-        
-        Args:
-            debugging_port: The debugging port to look for
-            config_file: Path to the config file
-            logger: Optional logger for recording events
-            
-        Returns:
-            dict: Browser information or None if no running browser is configured for this port
-        """
-        if not os.path.exists(config_file):
-            return None
-            
-        try:
-            with open(config_file, 'r') as f:
-                browser_info_dict = json.load(f)
-            
-            # Get browser info from port map
-            if isinstance(browser_info_dict, dict) and "port_map" in browser_info_dict:
-                port_str = str(debugging_port)
-                if port_str in browser_info_dict["port_map"]:
-                    browser_info = browser_info_dict["port_map"][port_str]
-                    
-                    # Check if the browser is still running
-                    pids = browser_info.get('pid', '')
-                    if isinstance(pids, str):
-                        pids = [int(pid) for pid in pids.split() if pid.isdigit()]
-                    elif isinstance(pids, int):
-                        pids = [pids]
-                    else:
-                        pids = []
-
-                    # Check if any of the PIDs are running
-                    if not pids:
-                        if logger:
-                            logger.warning(f"Built-in browser on port {debugging_port} has no valid PID", tag="BUILTIN")
-                        # Remove this port from the dictionary
-                        del browser_info_dict["port_map"][port_str]
-                        with open(config_file, 'w') as f:
-                            json.dump(browser_info_dict, f, indent=2)
-                        return None
-                    # Check if any of the PIDs are running
-                    for pid in pids:
-                        if is_browser_running(pid):
-                            browser_info['pid'] = pid
-                            break
-                    else:
-                        # If none of the PIDs are running, remove this port from the dictionary
-                        if logger:
-                            logger.warning(f"Built-in browser on port {debugging_port} is not running", tag="BUILTIN")
-                        # Remove this port from the dictionary
-                        del browser_info_dict["port_map"][port_str]
-                        with open(config_file, 'w') as f:
-                            json.dump(browser_info_dict, f, indent=2)
-                        return None
-                    
-                    return browser_info
-            
-            return None
-                
-        except Exception as e:
-            if logger:
-                logger.error(f"Error reading built-in browser config: {str(e)}", tag="BUILTIN")
-            return None
-            
-    def get_browser_info(self) -> Optional[Dict[str, Any]]:
-        """Get information about the current built-in browser instance.
-        
-        Returns:
-            dict: Browser information or None if no running browser is configured
-        """
-        return self._get_builtin_browser_info(
-            debugging_port=self.config.debugging_port,
-            config_file=self.builtin_config_file,
-            logger=self.logger
-        )
-    
-    async def launch_builtin_browser(self, 
-                               browser_type: str = "chromium",
-                               debugging_port: int = 9222,
-                               headless: bool = True) -> Optional[str]:
-        """Launch a browser in the background for use as the built-in browser.
-        
-        Args:
-            browser_type: Type of browser to launch ('chromium' or 'firefox')
-            debugging_port: Port to use for CDP debugging
-            headless: Whether to run in headless mode
-            
-        Returns:
-            str: CDP URL for the browser, or None if launch failed
-        """
-        # Check if there's an existing browser still running
-        browser_info = self._get_builtin_browser_info(
-            debugging_port=debugging_port,
-            config_file=self.builtin_config_file,
-            logger=self.logger
-        )
-        if browser_info:
-            if self.logger:
-                self.logger.info(f"Built-in browser is already running on port {debugging_port}", tag="BUILTIN")
-            return browser_info.get('cdp_url')
-        
-        # Create a user data directory for the built-in browser
-        user_data_dir = os.path.join(self.builtin_browser_dir, "user_data")
-     
-        # Raise error if user data dir is already engaged
-        if self._check_user_dir_is_engaged(user_data_dir):
-            raise Exception(f"User data directory {user_data_dir} is already engaged by another browser instance.")
-            
-        # Create the user data directory if it doesn't exist
-        os.makedirs(user_data_dir, exist_ok=True)
-        
-        # Prepare browser launch arguments
-        browser_args = super()._build_browser_args()
-        browser_path = await get_browser_executable(browser_type)
-        base_args = [browser_path]
-
-        if browser_type == "chromium":
-            args = [
-                browser_path,
-                f"--remote-debugging-port={debugging_port}",
-                f"--user-data-dir={user_data_dir}",
-            ]
-            # if headless:
-            #     args.append("--headless=new")
-
-        elif browser_type == "firefox":
-            args = [
-                browser_path,
-                "--remote-debugging-port",
-                str(debugging_port),
-                "--profile",
-                user_data_dir,
-            ]
-            if headless:
-                args.append("--headless")
-        else:
-            if self.logger:
-                self.logger.error(f"Browser type {browser_type} not supported for built-in browser", tag="BUILTIN")
-            return None
-        
-        args = base_args + browser_args + args
-        
-        try:
-
-            # Check if the port is already in use
-            PID = ""
-            cdp_url = f"http://localhost:{debugging_port}"
-            config_json = await self._check_port_in_use(cdp_url)
-            if config_json:
-                if self.logger:
-                    self.logger.info(f"Port {debugging_port} is already in use.", tag="BUILTIN")
-                PID = find_process_by_port(debugging_port)
-            else:
-                # Start the browser process detached
-                process = None
-                if is_windows():
-                    process = subprocess.Popen(
-                        args, 
-                        stdout=subprocess.PIPE, 
-                        stderr=subprocess.PIPE,
-                        creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
-                    )
-                else:
-                    process = subprocess.Popen(
-                        args, 
-                        stdout=subprocess.PIPE, 
-                        stderr=subprocess.PIPE,
-                        preexec_fn=os.setpgrp  # Start in a new process group
-                    )
-                
-                # Wait briefly to ensure the process starts successfully
-                await asyncio.sleep(2.0)
-                
-                # Check if the process is still running
-                if process and process.poll() is not None:
-                    if self.logger:
-                        self.logger.error(f"Browser process exited immediately with code {process.returncode}", tag="BUILTIN")
-                    return None
-            
-                PID = process.pid
-                # Construct CDP URL
-                config_json = await self._check_port_in_use(cdp_url)
-
-            
-            # Create browser info
-            browser_info = {
-                'pid': PID,
-                'cdp_url': cdp_url,
-                'user_data_dir': user_data_dir,
-                'browser_type': browser_type,
-                'debugging_port': debugging_port,
-                'start_time': time.time(),
-                'config': config_json
-            }
-            
-            # Read existing config file if it exists
-            port_map = {}
-            if os.path.exists(self.builtin_config_file):
-                try:
-                    with open(self.builtin_config_file, 'r') as f:
-                        existing_data = json.load(f)
-                    
-                    # Check if it already uses port mapping
-                    if isinstance(existing_data, dict) and "port_map" in existing_data:
-                        port_map = existing_data["port_map"]
-
-                    # # Convert legacy format to port mapping
-                    # elif isinstance(existing_data, dict) and "debugging_port" in existing_data:
-                    #     old_port = str(existing_data.get("debugging_port"))
-                    #     if self._is_browser_running(existing_data.get("pid")):
-                    #         port_map[old_port] = existing_data
-                except Exception as e:
-                    if self.logger:
-                        self.logger.warning(f"Could not read existing config: {str(e)}", tag="BUILTIN")
-            
-            # Add/update this browser in the port map
-            port_map[str(debugging_port)] = browser_info
-            
-            # Write updated config
-            with open(self.builtin_config_file, 'w') as f:
-                json.dump({"port_map": port_map}, f, indent=2)
-                
-            # Detach from the browser process - don't keep any references
-            # This is important to allow the Python script to exit while the browser continues running
-            process = None
-                
-            if self.logger:
-                self.logger.success(f"Built-in browser launched at CDP URL: {cdp_url}", tag="BUILTIN")
-            return cdp_url
-            
-        except Exception as e:
-            if self.logger:
-                self.logger.error(f"Error launching built-in browser: {str(e)}", tag="BUILTIN")
-            return None
-
-    async def _check_port_in_use(self, cdp_url: str) -> dict:
-        """Check if a port is already in use by a Chrome DevTools instance.
-        
-        Args:
-            cdp_url: The CDP URL to check
-            
-        Returns:
-            dict: Chrome DevTools protocol version information or None if not found
-        """
-        import aiohttp
-        json_url = f"{cdp_url}/json/version"
-        json_config = None
-        
-        try:
-            async with aiohttp.ClientSession() as session:
-                try:
-                    async with session.get(json_url, timeout=2.0) as response:
-                        if response.status == 200:
-                            json_config = await response.json()
-                            if self.logger:
-                                self.logger.debug(f"Found CDP server running at {cdp_url}", tag="BUILTIN")
-                            return json_config
-                except (aiohttp.ClientError, asyncio.TimeoutError):
-                    pass
-            return None
-        except Exception as e:
-            if self.logger:
-                self.logger.debug(f"Error checking CDP port: {str(e)}", tag="BUILTIN")
-            return None
-
-    async def kill_builtin_browser(self) -> bool:
-        """Kill the built-in browser if it's running.
-        
-        Returns:
-            bool: True if the browser was killed, False otherwise
-        """
-        browser_info = self.get_browser_info()
-        if not browser_info:
-            if self.logger:
-                self.logger.warning(f"No built-in browser found on port {self.config.debugging_port}", tag="BUILTIN")
-            return False
-            
-        pid = browser_info.get('pid')
-        if not pid:
-            return False
-            
-        success, error_msg = terminate_process(pid, logger=self.logger)
-        if success:
-            # Update config file to remove this browser
-            with open(self.builtin_config_file, 'r') as f:
-                browser_info_dict = json.load(f)
-
-            # Remove this port from the dictionary
-            port_str = str(self.config.debugging_port)
-            if port_str in browser_info_dict.get("port_map", {}):
-                del browser_info_dict["port_map"][port_str]
-
-            with open(self.builtin_config_file, 'w') as f:
-                json.dump(browser_info_dict, f, indent=2)
-
-            # Remove user data directory if it exists
-            if os.path.exists(self.builtin_browser_dir):
-                shutil.rmtree(self.builtin_browser_dir)
-
-            # Clear the browser info cache
-            self.browser = None
-            self.temp_dir = None
-            self.shutting_down = True
-                
-            if self.logger:
-                self.logger.success("Built-in browser terminated", tag="BUILTIN")
-            return True
-        else:
-            if self.logger:
-                self.logger.error(f"Error killing built-in browser: {error_msg}", tag="BUILTIN")
-            return False
-    
-    async def get_builtin_browser_status(self) -> Dict[str, Any]:
-        """Get status information about the built-in browser.
-        
-        Returns:
-            dict: Status information with running, cdp_url, and info fields
-        """
-        browser_info = self.get_browser_info()
-        
-        if not browser_info:
-            return {
-                'running': False,
-                'cdp_url': None,
-                'info': None,
-                'port': self.config.debugging_port
-            }
-            
-        return {
-            'running': True,
-            'cdp_url': browser_info.get('cdp_url'),
-            'info': browser_info,
-            'port': self.config.debugging_port
-        }
-
-    async def close(self):
-        """Close the built-in browser and clean up resources."""
-        # Call parent class close method
-        await super().close()
-        
-        # Clean up built-in browser if we created it and were in shutdown mode
-        if self.shutting_down:
-            await self.kill_builtin_browser()
-            if self.logger:
-                self.logger.debug("Killed built-in browser during shutdown", tag="BUILTIN")

crawl4ai/browser/strategies/cdp.py

@@ -1,281 +0,0 @@
-"""Browser strategies module for Crawl4AI.
-
-This module implements the browser strategy pattern for different
-browser implementations, including Playwright, CDP, and builtin browsers.
-"""
-
-import asyncio
-import os
-import time
-import json
-import subprocess
-import shutil
-from typing import Optional, Tuple, List
-
-from playwright.async_api import BrowserContext, Page
-
-from ...async_logger import AsyncLogger
-from ...async_configs import BrowserConfig, CrawlerRunConfig
-from ..utils import get_playwright, get_browser_executable, create_temp_directory, is_windows, check_process_is_running, terminate_process
-
-from .base import BaseBrowserStrategy
-
-class CDPBrowserStrategy(BaseBrowserStrategy):
-    """CDP-based browser strategy.
-    
-    This strategy connects to an existing browser using CDP protocol or
-    launches and connects to a browser using CDP.
-    """
-    
-    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
-        """Initialize the CDP browser strategy.
-        
-        Args:
-            config: Browser configuration
-            logger: Logger for recording events and errors
-        """
-        super().__init__(config, logger)
-        self.sessions = {}
-        self.session_ttl = 1800  # 30 minutes
-        self.browser_process = None
-        self.temp_dir = None
-        self.shutting_down = False
-        
-    async def start(self):
-        """Start or connect to the browser using CDP.
-        
-        Returns:
-            self: For method chaining
-        """
-        # Call the base class start to initialize Playwright
-        await super().start()
-        
-        try:
-            # Get or create CDP URL
-            cdp_url = await self._get_or_create_cdp_url()
-            
-            # Connect to the browser using CDP
-            self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url)
-            
-            # Get or create default context
-            contexts = self.browser.contexts
-            if contexts:
-                self.default_context = contexts[0]
-            else:
-                self.default_context = await self.create_browser_context()
-            
-            await self.setup_context(self.default_context)
-            
-            if self.logger:
-                self.logger.debug(f"Connected to CDP browser at {cdp_url}", tag="CDP")
-
-        except Exception as e:
-            if self.logger:
-                self.logger.error(f"Failed to connect to CDP browser: {str(e)}", tag="CDP")
-
-            # Clean up any resources before re-raising
-            await self._cleanup_process()
-            raise
-            
-        return self
-    
-    async def _get_or_create_cdp_url(self) -> str:
-        """Get existing CDP URL or launch a browser and return its CDP URL.
-        
-        Returns:
-            str: CDP URL for connecting to the browser
-        """
-        # If CDP URL is provided, just return it
-        if self.config.cdp_url:
-            return self.config.cdp_url
-
-        # Create temp dir if needed
-        if not self.config.user_data_dir:
-            self.temp_dir = create_temp_directory()
-            user_data_dir = self.temp_dir
-        else:
-            user_data_dir = self.config.user_data_dir
-
-        # Get browser args based on OS and browser type
-        # args = await self._get_browser_args(user_data_dir)
-        browser_args = super()._build_browser_args()
-        browser_path = await get_browser_executable(self.config.browser_type)
-        base_args = [browser_path]
-
-        if self.config.browser_type == "chromium":
-            args = [
-                f"--remote-debugging-port={self.config.debugging_port}",
-                f"--user-data-dir={user_data_dir}",
-            ]
-            # if self.config.headless:
-            #     args.append("--headless=new")
-
-        elif self.config.browser_type == "firefox":
-            args = [
-                "--remote-debugging-port",
-                str(self.config.debugging_port),
-                "--profile",
-                user_data_dir,
-            ]
-            if self.config.headless:
-                args.append("--headless")
-        else:
-            raise NotImplementedError(f"Browser type {self.config.browser_type} not supported")
-
-        args = base_args + browser_args['args'] + args         
-
-        # Start browser process
-        try:
-            # Use DETACHED_PROCESS flag on Windows to fully detach the process
-            # On Unix, we'll use preexec_fn=os.setpgrp to start the process in a new process group
-            if is_windows():
-                self.browser_process = subprocess.Popen(
-                    args, 
-                    stdout=subprocess.PIPE, 
-                    stderr=subprocess.PIPE,
-                    creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
-                )
-            else:
-                self.browser_process = subprocess.Popen(
-                    args, 
-                    stdout=subprocess.PIPE, 
-                    stderr=subprocess.PIPE,
-                    preexec_fn=os.setpgrp  # Start in a new process group
-                )
-                
-            # Monitor for a short time to make sure it starts properly
-            is_running, return_code, stdout, stderr = await check_process_is_running(self.browser_process, delay=2)
-            if not is_running:
-                if self.logger:
-                    self.logger.error(
-                        message="Browser process terminated unexpectedly | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}",
-                        tag="ERROR",
-                        params={
-                            "code": return_code,
-                            "stdout": stdout.decode() if stdout else "",
-                            "stderr": stderr.decode() if stderr else "",
-                        },
-                    )
-                await self._cleanup_process()
-                raise Exception("Browser process terminated unexpectedly")
-
-            return f"http://localhost:{self.config.debugging_port}"
-        except Exception as e:
-            await self._cleanup_process()
-            raise Exception(f"Failed to start browser: {e}")    
-
-    async def _cleanup_process(self):
-        """Cleanup browser process and temporary directory."""
-        # Set shutting_down flag BEFORE any termination actions
-        self.shutting_down = True
-
-        if self.browser_process:
-            try:
-                # Only attempt termination if the process is still running
-                if self.browser_process.poll() is None:
-                    # Use our robust cross-platform termination utility
-                    success = terminate_process(
-                        pid=self.browser_process.pid,
-                        timeout=1.0,  # Equivalent to the previous 10*0.1s wait
-                        logger=self.logger
-                    )
-                    
-                    if not success and self.logger:
-                        self.logger.warning(
-                            message="Failed to terminate browser process cleanly",
-                            tag="PROCESS"
-                        )
-                        
-            except Exception as e:
-                if self.logger:
-                    self.logger.error(
-                        message="Error during browser process cleanup: {error}",
-                        tag="ERROR",
-                        params={"error": str(e)},
-                    )
-
-        if self.temp_dir and os.path.exists(self.temp_dir):
-            try:
-                shutil.rmtree(self.temp_dir)
-                self.temp_dir = None
-                if self.logger:
-                    self.logger.debug("Removed temporary directory", tag="CDP")
-            except Exception as e:
-                if self.logger:
-                    self.logger.error(
-                        message="Error removing temporary directory: {error}",
-                        tag="CDP",
-                        params={"error": str(e)}
-                    )
-
-        self.browser_process = None
-    
-    async def _generate_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
-        # For CDP, we typically use the shared default_context
-        context = self.default_context
-        pages = context.pages
-        
-        # Otherwise, check if we have an existing context for this config
-        config_signature = self._make_config_signature(crawlerRunConfig)
-        self.contexts_by_config[config_signature] = context
-
-        await self.setup_context(context, crawlerRunConfig)
-
-        # Check if there's already a page with the target URL
-        page = next((p for p in pages if p.url == crawlerRunConfig.url), None)
-        
-        # If not found, create a new page
-        if not page:
-            page = await context.new_page()
-        
-        return page, context
-
-    async def _get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
-        """Get a page for the given configuration.
-        
-        Args:
-            crawlerRunConfig: Configuration object for the crawler run
-            
-        Returns:
-            Tuple of (Page, BrowserContext)
-        """
-        # Call parent method to ensure browser is started
-        await super().get_page(crawlerRunConfig)
-        
-        # For CDP, we typically use the shared default_context
-        context = self.default_context
-        pages = context.pages
-        
-        # Otherwise, check if we have an existing context for this config
-        config_signature = self._make_config_signature(crawlerRunConfig)
-        self.contexts_by_config[config_signature] = context
-
-        await self.setup_context(context, crawlerRunConfig)
-
-        # Check if there's already a page with the target URL
-        page = next((p for p in pages if p.url == crawlerRunConfig.url), None)
-        
-        # If not found, create a new page
-        if not page:
-            page = await context.new_page()
-        
-        # If a session_id is specified, store this session for reuse
-        if crawlerRunConfig.session_id:
-            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
-        
-        return page, context
-
-    async def close(self):
-        """Close the CDP browser and clean up resources."""
-        # Skip cleanup if using external CDP URL and not launched by us
-        if self.config.cdp_url and not self.browser_process:
-            if self.logger:
-                self.logger.debug("Skipping cleanup for external CDP browser", tag="CDP")
-            return
-        
-        # Call parent implementation for common cleanup
-        await super().close()
-        
-        # Additional CDP-specific cleanup
-        await asyncio.sleep(0.5)
-        await self._cleanup_process()

crawl4ai/browser/strategies/docker_strategy.py

@@ -1,430 +0,0 @@
-"""Docker browser strategy module for Crawl4AI.
-
-This module provides browser strategies for running browsers in Docker containers,
-which offers better isolation, consistency across platforms, and easy scaling.
-"""
-
-import os
-import uuid
-from typing import List, Optional
-
-
-from ...async_logger import AsyncLogger
-from ...async_configs import BrowserConfig
-from ..models import DockerConfig
-from ..docker_registry import DockerRegistry
-from ..docker_utils import DockerUtils
-from .builtin import CDPBrowserStrategy
-from .base import BaseBrowserStrategy
-
-class DockerBrowserStrategy(CDPBrowserStrategy):
-    """Docker-based browser strategy.
-
-    Extends the CDPBrowserStrategy to run browsers in Docker containers.
-    Supports two modes:
-    1. "connect" - Uses a Docker image with Chrome already running
-    2. "launch" - Starts Chrome within the container with custom settings
-
-    Attributes:
-        docker_config: Docker-specific configuration options
-        container_id: ID of current Docker container
-        container_name: Name assigned to the container
-        registry: Registry for tracking and reusing containers
-        docker_utils: Utilities for Docker operations
-        chrome_process_id: Process ID of Chrome within container
-        socat_process_id: Process ID of socat within container
-        internal_cdp_port: Chrome's internal CDP port
-        internal_mapped_port: Port that socat maps to internally
-    """
-
-    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
-        """Initialize the Docker browser strategy.
-
-        Args:
-            config: Browser configuration including Docker-specific settings
-            logger: Logger for recording events and errors
-        """
-        super().__init__(config, logger)
-
-        # Initialize Docker-specific attributes
-        self.docker_config = self.config.docker_config or DockerConfig()
-        self.container_id = None
-        self.container_name = f"crawl4ai-browser-{uuid.uuid4().hex[:8]}"
-
-        # Use the shared registry file path for consistency with BuiltinBrowserStrategy
-        registry_file = self.docker_config.registry_file
-        if registry_file is None and self.config.user_data_dir:
-            # Use the same registry file as BuiltinBrowserStrategy if possible
-            registry_file = os.path.join(
-                os.path.dirname(self.config.user_data_dir), "browser_config.json"
-            )
-
-        self.registry = DockerRegistry(self.docker_config.registry_file)
-        self.docker_utils = DockerUtils(logger)
-        self.chrome_process_id = None
-        self.socat_process_id = None
-        self.internal_cdp_port = 9222  # Chrome's internal CDP port
-        self.internal_mapped_port = 9223  # Port that socat maps to internally
-        self.shutting_down = False
-
-    async def start(self):
-        """Start or connect to a browser running in a Docker container.
-
-        This method initializes Playwright and establishes a connection to
-        a browser running in a Docker container. Depending on the configured mode:
-        - "connect": Connects to a container with Chrome already running
-        - "launch": Creates a container and launches Chrome within it
-
-        Returns:
-            self: For method chaining
-        """
-        # Initialize Playwright
-        await BaseBrowserStrategy.start(self)
-
-        if self.logger:
-            self.logger.info(
-                f"Starting Docker browser strategy in {self.docker_config.mode} mode",
-                tag="DOCKER",
-            )
-
-        try:
-            # Get CDP URL by creating or reusing a Docker container
-            # This handles the container management and browser startup
-            cdp_url = await self._get_or_create_cdp_url()
-
-            if not cdp_url:
-                raise Exception(
-                    "Failed to establish CDP connection to Docker container"
-                )
-
-            if self.logger:
-                self.logger.info(
-                    f"Connecting to browser in Docker via CDP: {cdp_url}", tag="DOCKER"
-                )
-
-            # Connect to the browser using CDP
-            self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url)
-
-            # Get existing context or create default context
-            contexts = self.browser.contexts
-            if contexts:
-                self.default_context = contexts[0]
-                if self.logger:
-                    self.logger.debug("Using existing browser context", tag="DOCKER")
-            else:
-                if self.logger:
-                    self.logger.debug("Creating new browser context", tag="DOCKER")
-                self.default_context = await self.create_browser_context()
-                await self.setup_context(self.default_context)
-
-            return self
-
-        except Exception as e:
-            # Clean up resources if startup fails
-            if self.container_id and not self.docker_config.persistent:
-                if self.logger:
-                    self.logger.warning(
-                        f"Cleaning up container after failed start: {self.container_id[:12]}",
-                        tag="DOCKER",
-                    )
-                await self.docker_utils.remove_container(self.container_id)
-                self.registry.unregister_container(self.container_id)
-                self.container_id = None
-
-            if self.playwright:
-                await self.playwright.stop()
-                self.playwright = None
-
-            # Re-raise the exception
-            if self.logger:
-                self.logger.error(
-                    f"Failed to start Docker browser: {str(e)}", tag="DOCKER"
-                )
-            raise
-
-    async def _generate_config_hash(self) -> str:
-        """Generate a hash of the configuration for container matching.
-
-        Returns:
-            Hash string uniquely identifying this configuration
-        """
-        # Create a dict with the relevant parts of the config
-        config_dict = {
-            "image": self.docker_config.image,
-            "mode": self.docker_config.mode,
-            "browser_type": self.config.browser_type,
-            "headless": self.config.headless,
-        }
-
-        # Add browser-specific config if in launch mode
-        if self.docker_config.mode == "launch":
-            config_dict.update(
-                {
-                    "text_mode": self.config.text_mode,
-                    "light_mode": self.config.light_mode,
-                    "viewport_width": self.config.viewport_width,
-                    "viewport_height": self.config.viewport_height,
-                }
-            )
-
-        # Use the utility method to generate the hash
-        return self.docker_utils.generate_config_hash(config_dict)
-
-    async def _get_or_create_cdp_url(self) -> str:
-        """Get CDP URL by either creating a new container or using an existing one.
-
-        Returns:
-            CDP URL for connecting to the browser
-
-        Raises:
-            Exception: If container creation or browser launch fails
-        """
-        # If CDP URL is explicitly provided, use it
-        if self.config.cdp_url:
-            return self.config.cdp_url
-
-        # Ensure Docker image exists (will build if needed)
-        image_name = await self.docker_utils.ensure_docker_image_exists(
-            self.docker_config.image, self.docker_config.mode
-        )
-
-        # Generate config hash for container matching
-        config_hash = await self._generate_config_hash()
-
-        # Look for existing container with matching config
-        container_id = await self.registry.find_container_by_config(
-            config_hash, self.docker_utils
-        )
-
-        if container_id:
-            # Use existing container
-            self.container_id = container_id
-            host_port = self.registry.get_container_host_port(container_id)
-            if self.logger:
-                self.logger.info(
-                    f"Using existing Docker container: {container_id[:12]}",
-                    tag="DOCKER",
-                )
-        else:
-            # Get a port for the new container
-            host_port = (
-                self.docker_config.host_port
-                or self.registry.get_next_available_port(self.docker_utils)
-            )
-
-            # Prepare volumes list
-            volumes = list(self.docker_config.volumes)
-
-            # Add user data directory if specified
-            if self.docker_config.user_data_dir:
-                # Ensure user data directory exists
-                os.makedirs(self.docker_config.user_data_dir, exist_ok=True)
-                volumes.append(
-                    f"{self.docker_config.user_data_dir}:{self.docker_config.container_user_data_dir}"
-                )
-
-                # # Update config user_data_dir to point to container path
-                # self.config.user_data_dir = self.docker_config.container_user_data_dir
-
-            # Create a new container
-            container_id = await self.docker_utils.create_container(
-                image_name=image_name,
-                host_port=host_port,
-                container_name=self.container_name,
-                volumes=volumes,
-                network=self.docker_config.network,
-                env_vars=self.docker_config.env_vars,
-                cpu_limit=self.docker_config.cpu_limit,
-                memory_limit=self.docker_config.memory_limit,
-                extra_args=self.docker_config.extra_args,
-            )
-
-            if not container_id:
-                raise Exception("Failed to create Docker container")
-
-            self.container_id = container_id
-
-            # Wait for container to be ready
-            await self.docker_utils.wait_for_container_ready(container_id)
-
-            # Handle specific setup based on mode
-            if self.docker_config.mode == "launch":
-                # In launch mode, we need to start socat and Chrome
-                await self.docker_utils.start_socat_in_container(container_id)
-
-                # Build browser arguments
-                browser_args = self._build_browser_args()
-
-                # Launch Chrome
-                await self.docker_utils.launch_chrome_in_container(
-                    container_id, browser_args
-                )
-
-                # Get PIDs for later cleanup
-                self.chrome_process_id = (
-                    await self.docker_utils.get_process_id_in_container(
-                        container_id, "chromium"
-                    )
-                )
-                self.socat_process_id = (
-                    await self.docker_utils.get_process_id_in_container(
-                        container_id, "socat"
-                    )
-                )
-
-            # Wait for CDP to be ready
-            cdp_json_config = await self.docker_utils.wait_for_cdp_ready(host_port)
-
-            if cdp_json_config:
-                # Register the container in the shared registry
-                self.registry.register_container(
-                    container_id, host_port, config_hash, cdp_json_config
-                )
-            else:
-                raise Exception("Failed to get CDP JSON config from Docker container")
-
-            if self.logger:
-                self.logger.success(
-                    f"Docker container ready: {container_id[:12]} on port {host_port}",
-                    tag="DOCKER",
-                )
-
-        # Return CDP URL
-        return f"http://localhost:{host_port}"
-
-    def _build_browser_args(self) -> List[str]:
-        """Build Chrome command line arguments based on BrowserConfig.
-
-        Returns:
-            List of command line arguments for Chrome
-        """
-        # Call parent method to get common arguments
-        browser_args = super()._build_browser_args()
-        return browser_args["args"] + [
-            f"--remote-debugging-port={self.internal_cdp_port}",
-            "--remote-debugging-address=0.0.0.0",  # Allow external connections
-            "--disable-dev-shm-usage",
-            "--headless=new",
-        ]
-
-        # args = [
-        #     "--no-sandbox",
-        #     "--disable-gpu",
-        #     f"--remote-debugging-port={self.internal_cdp_port}",
-        #     "--remote-debugging-address=0.0.0.0",  # Allow external connections
-        #     "--disable-dev-shm-usage",
-        # ]
-
-        # if self.config.headless:
-        #     args.append("--headless=new")
-
-        # if self.config.viewport_width and self.config.viewport_height:
-        #     args.append(f"--window-size={self.config.viewport_width},{self.config.viewport_height}")
-
-        # if self.config.user_agent:
-        #     args.append(f"--user-agent={self.config.user_agent}")
-
-        # if self.config.text_mode:
-        #     args.extend([
-        #         "--blink-settings=imagesEnabled=false",
-        #         "--disable-remote-fonts",
-        #         "--disable-images",
-        #         "--disable-javascript",
-        #     ])
-
-        # if self.config.light_mode:
-        #     # Import here to avoid circular import
-        #     from ..utils import get_browser_disable_options
-        #     args.extend(get_browser_disable_options())
-
-        # if self.config.user_data_dir:
-        #     args.append(f"--user-data-dir={self.config.user_data_dir}")
-
-        # if self.config.extra_args:
-        #     args.extend(self.config.extra_args)
-
-        # return args
-
-    async def close(self):
-        """Close the browser and clean up Docker container if needed."""
-        # Set flag to track if we were the ones initiating shutdown
-        initiated_shutdown = not self.shutting_down
-        # Storage persistence for Docker needs special handling
-        # We need to store state before calling super().close() which will close the browser
-        if (
-            self.browser
-            and self.docker_config.user_data_dir
-            and self.docker_config.persistent
-        ):
-            for context in self.browser.contexts:
-                try:
-                    # Ensure directory exists
-                    os.makedirs(self.docker_config.user_data_dir, exist_ok=True)
-
-                    # Save storage state to user data directory
-                    storage_path = os.path.join(
-                        self.docker_config.user_data_dir, "storage_state.json"
-                    )
-                    await context.storage_state(path=storage_path)
-                    if self.logger:
-                        self.logger.debug(
-                            "Persisted Docker-specific storage state", tag="DOCKER"
-                        )
-                except Exception as e:
-                    if self.logger:
-                        self.logger.warning(
-                            message="Failed to persist Docker storage state: {error}",
-                            tag="DOCKER",
-                            params={"error": str(e)},
-                        )
-
-        # Call parent method to handle common cleanup
-        await super().close()
-
-        # Only perform container cleanup if we initiated shutdown
-        # and we need to handle Docker-specific resources
-        if initiated_shutdown:
-            # Only clean up container if not persistent
-            if self.container_id and not self.docker_config.persistent:
-                # Stop Chrome process in "launch" mode
-                if self.docker_config.mode == "launch" and self.chrome_process_id:
-                    await self.docker_utils.stop_process_in_container(
-                        self.container_id, self.chrome_process_id
-                    )
-                    if self.logger:
-                        self.logger.debug(
-                            f"Stopped Chrome process {self.chrome_process_id} in container",
-                            tag="DOCKER",
-                        )
-
-                # Stop socat process in "launch" mode
-                if self.docker_config.mode == "launch" and self.socat_process_id:
-                    await self.docker_utils.stop_process_in_container(
-                        self.container_id, self.socat_process_id
-                    )
-                    if self.logger:
-                        self.logger.debug(
-                            f"Stopped socat process {self.socat_process_id} in container",
-                            tag="DOCKER",
-                        )
-
-                # Remove or stop container based on configuration
-                if self.docker_config.remove_on_exit:
-                    await self.docker_utils.remove_container(self.container_id)
-                    # Unregister from registry
-                    if hasattr(self, "registry") and self.registry:
-                        self.registry.unregister_container(self.container_id)
-                    if self.logger:
-                        self.logger.debug(
-                            f"Removed Docker container {self.container_id}",
-                            tag="DOCKER",
-                        )
-                else:
-                    await self.docker_utils.stop_container(self.container_id)
-                    if self.logger:
-                        self.logger.debug(
-                            f"Stopped Docker container {self.container_id}",
-                            tag="DOCKER",
-                        )
-
-                self.container_id = None

crawl4ai/browser/strategies/playwright.py

@@ -1,134 +0,0 @@
-"""Browser strategies module for Crawl4AI.
-
-This module implements the browser strategy pattern for different
-browser implementations, including Playwright, CDP, and builtin browsers.
-"""
-
-import time
-from typing import Optional, Tuple
-
-from playwright.async_api import BrowserContext, Page
-
-from ...async_logger import AsyncLogger
-from ...async_configs import BrowserConfig, CrawlerRunConfig
-
-from playwright_stealth import StealthConfig
-
-from .base import BaseBrowserStrategy
-
-stealth_config = StealthConfig(
-    webdriver=True,
-    chrome_app=True,
-    chrome_csi=True,
-    chrome_load_times=True,
-    chrome_runtime=True,
-    navigator_languages=True,
-    navigator_plugins=True,
-    navigator_permissions=True,
-    webgl_vendor=True,
-    outerdimensions=True,
-    navigator_hardware_concurrency=True,
-    media_codecs=True,
-)
-
-class PlaywrightBrowserStrategy(BaseBrowserStrategy):
-    """Standard Playwright browser strategy.
-    
-    This strategy launches a new browser instance using Playwright
-    and manages browser contexts.
-    """
-    
-    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
-        """Initialize the Playwright browser strategy.
-        
-        Args:
-            config: Browser configuration
-            logger: Logger for recording events and errors
-        """
-        super().__init__(config, logger)
-        # No need to re-initialize sessions and session_ttl as they're now in the base class
-    
-    async def start(self):
-        """Start the browser instance.
-        
-        Returns:
-            self: For method chaining
-        """
-        # Call the base class start to initialize Playwright
-        await super().start()
-        
-        # Build browser arguments using the base class method
-        browser_args = self._build_browser_args()
-        
-        try:
-            # Launch appropriate browser type
-            if self.config.browser_type == "firefox":
-                self.browser = await self.playwright.firefox.launch(**browser_args)
-            elif self.config.browser_type == "webkit":
-                self.browser = await self.playwright.webkit.launch(**browser_args)
-            else:
-                self.browser = await self.playwright.chromium.launch(**browser_args)
-                
-            self.default_context = self.browser
-            
-            if self.logger:
-                self.logger.debug(f"Launched {self.config.browser_type} browser", tag="BROWSER")
-                
-        except Exception as e:
-            if self.logger:
-                self.logger.error(f"Failed to launch browser: {str(e)}", tag="BROWSER")
-            raise
-            
-        return self
-
-    async def _generate_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
-        # Otherwise, check if we have an existing context for this config
-        config_signature = self._make_config_signature(crawlerRunConfig)
-        
-        async with self._contexts_lock:
-            if config_signature in self.contexts_by_config:
-                context = self.contexts_by_config[config_signature]
-            else:
-                # Create and setup a new context
-                context = await self.create_browser_context(crawlerRunConfig)
-                await self.setup_context(context, crawlerRunConfig)
-                self.contexts_by_config[config_signature] = context
-
-        # Create a new page from the chosen context
-        page = await context.new_page()
-        
-        return page, context
-
-    async def _get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
-        """Get a page for the given configuration.
-        
-        Args:
-            crawlerRunConfig: Configuration object for the crawler run
-            
-        Returns:
-            Tuple of (Page, BrowserContext)
-        """
-        # Call parent method to ensure browser is started
-        await super().get_page(crawlerRunConfig)
-        
-        # Otherwise, check if we have an existing context for this config
-        config_signature = self._make_config_signature(crawlerRunConfig)
-        
-        async with self._contexts_lock:
-            if config_signature in self.contexts_by_config:
-                context = self.contexts_by_config[config_signature]
-            else:
-                # Create and setup a new context
-                context = await self.create_browser_context(crawlerRunConfig)
-                await self.setup_context(context, crawlerRunConfig)
-                self.contexts_by_config[config_signature] = context
-
-        # Create a new page from the chosen context
-        page = await context.new_page()
-        
-        # If a session_id is specified, store this session so we can reuse later
-        if crawlerRunConfig.session_id:
-            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
-            
-        return page, context
-        

crawl4ai/browser/utils.py

@@ -11,9 +11,7 @@ import sys
 import time
 import tempfile
 import subprocess
-from typing import Optional, Tuple, Union
-import signal
-import psutil
+from typing import Optional
 
 from playwright.async_api import async_playwright
 
@@ -95,8 +93,6 @@ def is_browser_running(pid: Optional[int]) -> bool:
         return False
         
     try:
-        if type(pid) is str:
-            pid = int(pid)
         # Check if the process exists
         if is_windows():
             process = subprocess.run(["tasklist", "/FI", f"PID eq {pid}"], 
@@ -330,136 +326,3 @@ async def find_optimal_browser_config(total_urls=50, verbose=True, rate_limit_de
         "optimal": optimal,
         "all_configs": results
     }
-
-
-# Find process ID of the existing browser using os
-def find_process_by_port(port: int) -> str:
-    """Find process ID listening on a specific port.
-    
-    Args:
-        port: Port number to check
-        
-    Returns:
-        str: Process ID or empty string if not found
-    """
-    try:
-        if is_windows():
-            cmd = f"netstat -ano | findstr :{port}"
-            result = subprocess.check_output(cmd, shell=True).decode()
-            return result.strip().split()[-1] if result else ""
-        else:
-            cmd = f"lsof -i :{port} -t"
-            return subprocess.check_output(cmd, shell=True).decode().strip()
-    except subprocess.CalledProcessError:
-        return ""
-    
-async def check_process_is_running(process: subprocess.Popen, delay: float = 0.5) -> Tuple[bool, Optional[int], bytes, bytes]:
-    """Perform a quick check to make sure the browser started successfully."""
-    if not process:
-        return False, None, b"", b""
-        
-    # Check that process started without immediate termination
-    await asyncio.sleep(delay)
-    if process.poll() is not None:
-        # Process already terminated
-        stdout, stderr = b"", b""
-        try:
-            stdout, stderr = process.communicate(timeout=0.5)
-        except subprocess.TimeoutExpired:
-            pass
-
-        return False, process.returncode, stdout, stderr
-            
-
-    return True, 0, b"", b""
-
-
-def terminate_process(
-    pid: Union[int, str], 
-    timeout: float = 5.0,
-    force_kill_timeout: float = 3.0,
-    logger = None
-) -> Tuple[bool, Optional[str]]:
-    """
-    Robustly terminate a process across platforms with verification.
-    
-    Args:
-        pid: Process ID to terminate (int or string)
-        timeout: Seconds to wait for graceful termination before force killing
-        force_kill_timeout: Seconds to wait after force kill before considering it failed
-        logger: Optional logger object with error, warning, and info methods
-        
-    Returns:
-        Tuple of (success: bool, error_message: Optional[str])
-    """
-    # Convert pid to int if it's a string
-    if isinstance(pid, str):
-        try:
-            pid = int(pid)
-        except ValueError:
-            error_msg = f"Invalid PID format: {pid}"
-            if logger:
-                logger.error(error_msg)
-            return False, error_msg
-    
-    # Check if process exists
-    if not psutil.pid_exists(pid):
-        return True, None  # Process already terminated
-    
-    try:
-        process = psutil.Process(pid)
-        
-        # First attempt: graceful termination
-        if logger:
-            logger.info(f"Attempting graceful termination of process {pid}")
-            
-        if os.name == 'nt':  # Windows
-            subprocess.run(["taskkill", "/PID", str(pid)], 
-                          stdout=subprocess.DEVNULL, 
-                          stderr=subprocess.DEVNULL, 
-                          check=False)
-        else:  # Unix/Linux/MacOS
-            process.send_signal(signal.SIGTERM)
-        
-        # Wait for process to terminate
-        try:
-            process.wait(timeout=timeout)
-            if logger:
-                logger.info(f"Process {pid} terminated gracefully")
-            return True, None
-        except psutil.TimeoutExpired:
-            if logger:
-                logger.warning(f"Process {pid} did not terminate gracefully within {timeout} seconds, forcing termination")
-        
-        # Second attempt: force kill
-        if os.name == 'nt':  # Windows
-            subprocess.run(["taskkill", "/F", "/PID", str(pid)], 
-                          stdout=subprocess.DEVNULL, 
-                          stderr=subprocess.DEVNULL, 
-                          check=False)
-        else:  # Unix/Linux/MacOS
-            process.send_signal(signal.SIGKILL)
-        
-        # Verify process is killed
-        gone, alive = psutil.wait_procs([process], timeout=force_kill_timeout)
-        if process in alive:
-            error_msg = f"Failed to kill process {pid} even after force kill"
-            if logger:
-                logger.error(error_msg)
-            return False, error_msg
-            
-        if logger:
-            logger.info(f"Process {pid} terminated by force")
-        return True, None
-        
-    except psutil.NoSuchProcess:
-        # Process terminated while we were working with it
-        if logger:
-            logger.info(f"Process {pid} already terminated")
-        return True, None
-        
-    except Exception as e:
-        error_msg = f"Error terminating process {pid}: {str(e)}"
-        if logger:
-            logger.error(error_msg)
-        return False, error_msg

crawl4ai/cli.py

@@ -20,16 +20,13 @@ from crawl4ai import (
     BrowserConfig, 
     CrawlerRunConfig,
     LLMExtractionStrategy, 
-    LXMLWebScrapingStrategy,
     JsonCssExtractionStrategy,
     JsonXPathExtractionStrategy,
     BM25ContentFilter, 
     PruningContentFilter,
     BrowserProfiler,
-    DefaultMarkdownGenerator,
     LLMConfig
 )
-from crawl4ai.config import USER_SETTINGS
 from litellm import completion
 from pathlib import Path
 
@@ -178,12 +175,8 @@ def show_examples():
     # CSS-based extraction
     crwl https://example.com -e extract_css.yml -s css_schema.json -o json
 
-    # LLM-based extraction with config file
+    # LLM-based extraction
     crwl https://example.com -e extract_llm.yml -s llm_schema.json -o json
-    
-    # Quick LLM-based JSON extraction (prompts for LLM provider first time)
-    crwl https://example.com -j  # Auto-extracts structured data
-    crwl https://example.com -j "Extract product details including name, price, and features"  # With specific instructions
 
 3️⃣  Direct Parameters:
     # Browser settings
@@ -285,7 +278,7 @@ llm_schema.json:
     # Combine configs with direct parameters
     crwl https://example.com -B browser.yml -b "headless=false,viewport_width=1920"
 
-    # Full extraction pipeline with config files
+    # Full extraction pipeline
     crwl https://example.com \\
         -B browser.yml \\
         -C crawler.yml \\
@@ -293,12 +286,6 @@ llm_schema.json:
         -s llm_schema.json \\
         -o json \\
         -v
-        
-    # Quick LLM-based extraction with specific instructions
-    crwl https://amazon.com/dp/B01DFKC2SO \\
-        -j "Extract product title, current price, original price, rating, and all product specifications" \\
-        -b "headless=true,viewport_width=1280" \\
-        -v
 
     # Content filtering with BM25
     crwl https://example.com \\
@@ -340,14 +327,6 @@ For more documentation visit: https://github.com/unclecode/crawl4ai
       - google/gemini-pro
     
     See full list of providers: https://docs.litellm.ai/docs/providers
-    
-    # Set default LLM provider and token in advance
-    crwl config set DEFAULT_LLM_PROVIDER "anthropic/claude-3-sonnet"
-    crwl config set DEFAULT_LLM_PROVIDER_TOKEN "your-api-token-here"
-    
-    # Set default browser behavior
-    crwl config set BROWSER_HEADLESS false  # Always show browser window
-    crwl config set USER_AGENT_MODE random  # Use random user agent
 
 9️⃣ Profile Management:
     # Launch interactive profile manager
@@ -1004,19 +983,17 @@ def cdp_cmd(user_data_dir: Optional[str], port: int, browser_type: str, headless
 @click.option("--crawler-config", "-C", type=click.Path(exists=True), help="Crawler config file (YAML/JSON)")
 @click.option("--filter-config", "-f", type=click.Path(exists=True), help="Content filter config file")
 @click.option("--extraction-config", "-e", type=click.Path(exists=True), help="Extraction strategy config file")
-@click.option("--json-extract", "-j", is_flag=False, flag_value="", default=None, help="Extract structured data using LLM with optional description")
 @click.option("--schema", "-s", type=click.Path(exists=True), help="JSON schema for extraction")
 @click.option("--browser", "-b", type=str, callback=parse_key_values, help="Browser parameters as key1=value1,key2=value2")
 @click.option("--crawler", "-c", type=str, callback=parse_key_values, help="Crawler parameters as key1=value1,key2=value2")
 @click.option("--output", "-o", type=click.Choice(["all", "json", "markdown", "md", "markdown-fit", "md-fit"]), default="all")
-@click.option("--output-file", "-O", type=click.Path(), help="Output file path (default: stdout)")
-@click.option("--bypass-cache", "-b", is_flag=True, default=True, help="Bypass cache when crawling")
+@click.option("--bypass-cache", is_flag=True, default=True, help="Bypass cache when crawling")
 @click.option("--question", "-q", help="Ask a question about the crawled content")
 @click.option("--verbose", "-v", is_flag=True)
 @click.option("--profile", "-p", help="Use a specific browser profile (by name)")
 def crawl_cmd(url: str, browser_config: str, crawler_config: str, filter_config: str, 
-           extraction_config: str, json_extract: str, schema: str, browser: Dict, crawler: Dict,
-           output: str, output_file: str, bypass_cache: bool, question: str, verbose: bool, profile: str):
+           extraction_config: str, schema: str, browser: Dict, crawler: Dict,
+           output: str, bypass_cache: bool, question: str, verbose: bool, profile: str):
     """Crawl a website and extract content
     
     Simple Usage:
@@ -1060,65 +1037,21 @@ def crawl_cmd(url: str, browser_config: str, crawler_config: str, filter_config:
             crawler_cfg = crawler_cfg.clone(**crawler)
             
         # Handle content filter config
-        if filter_config or output in ["markdown-fit", "md-fit"]:
-            if filter_config:
-                filter_conf = load_config_file(filter_config)
-            elif not filter_config and output in ["markdown-fit", "md-fit"]:
-                filter_conf = {
-                    "type": "pruning",
-                    "query": "",
-                    "threshold": 0.48
-                }
+        if filter_config:
+            filter_conf = load_config_file(filter_config)
             if filter_conf["type"] == "bm25":
-                crawler_cfg.markdown_generator = DefaultMarkdownGenerator(
-                    content_filter = BM25ContentFilter(
-                        user_query=filter_conf.get("query"),
-                        bm25_threshold=filter_conf.get("threshold", 1.0)
-                    )
+                crawler_cfg.content_filter = BM25ContentFilter(
+                    user_query=filter_conf.get("query"),
+                    bm25_threshold=filter_conf.get("threshold", 1.0)
                 )
             elif filter_conf["type"] == "pruning":
-                crawler_cfg.markdown_generator = DefaultMarkdownGenerator(
-                    content_filter = PruningContentFilter(
-                        user_query=filter_conf.get("query"),
-                        threshold=filter_conf.get("threshold", 0.48)
-                    )
+                crawler_cfg.content_filter = PruningContentFilter(
+                    user_query=filter_conf.get("query"),
+                    threshold=filter_conf.get("threshold", 0.48)
                 )
-        
-        # Handle json-extract option (takes precedence over extraction-config)
-        if json_extract is not None:
-            # Get LLM provider and token
-            provider, token = setup_llm_config()
-            
-            # Default sophisticated instruction for structured data extraction
-            default_instruction = """Analyze the web page content and extract structured data as JSON. 
-If the page contains a list of items with repeated patterns, extract all items in an array. 
-If the page is an article or contains unique content, extract a comprehensive JSON object with all relevant information.
-Look at the content, intention of content, what it offers and find the data item(s) in the page.
-Always return valid, properly formatted JSON."""
-            
-            
-            default_instruction_with_user_query = """Analyze the web page content and extract structured data as JSON, following the below instruction and explanation of schema and always return valid, properly formatted JSON. \n\nInstruction:\n\n""" + json_extract
-            
-            # Determine instruction based on whether json_extract is empty or has content
-            instruction = default_instruction_with_user_query if json_extract else default_instruction
-            
-            # Create LLM extraction strategy
-            crawler_cfg.extraction_strategy = LLMExtractionStrategy(
-                llm_config=LLMConfig(provider=provider, api_token=token),
-                instruction=instruction,
-                schema=load_schema_file(schema),  # Will be None if no schema is provided
-                extraction_type="schema", #if schema else "block",
-                apply_chunking=False,
-                force_json_response=True,
-                verbose=verbose,
-            )
-            
-            # Set output to JSON if not explicitly specified
-            if output == "all":
-                output = "json"
                 
-        # Handle extraction strategy from config file (only if json-extract wasn't used)
-        elif extraction_config:
+        # Handle extraction strategy
+        if extraction_config:
             extract_conf = load_config_file(extraction_config)
             schema_data = load_schema_file(schema)
             
@@ -1152,13 +1085,6 @@ Always return valid, properly formatted JSON."""
         # No cache
         if bypass_cache:
             crawler_cfg.cache_mode = CacheMode.BYPASS
-
-        crawler_cfg.scraping_strategy = LXMLWebScrapingStrategy()    
-
-        config = get_global_config()
-        
-        browser_cfg.verbose = config.get("VERBOSE", False)
-        crawler_cfg.verbose = config.get("VERBOSE", False)
         
         # Run crawler
         result : CrawlResult = anyio.run(
@@ -1177,31 +1103,14 @@ Always return valid, properly formatted JSON."""
             return
         
         # Handle output
-        if not output_file:
-            if output == "all":
-                click.echo(json.dumps(result.model_dump(), indent=2))
-            elif output == "json":
-                print(result.extracted_content)
-                extracted_items = json.loads(result.extracted_content)
-                click.echo(json.dumps(extracted_items, indent=2))
-                
-            elif output in ["markdown", "md"]:
-                click.echo(result.markdown.raw_markdown)
-            elif output in ["markdown-fit", "md-fit"]:
-                click.echo(result.markdown.fit_markdown)
-        else:
-            if output == "all":
-                with open(output_file, "w") as f:
-                    f.write(json.dumps(result.model_dump(), indent=2))
-            elif output == "json":
-                with open(output_file, "w") as f:
-                    f.write(result.extracted_content)
-            elif output in ["markdown", "md"]:
-                with open(output_file, "w") as f:
-                    f.write(result.markdown.raw_markdown)
-            elif output in ["markdown-fit", "md-fit"]:
-                with open(output_file, "w") as f:
-                    f.write(result.markdown.fit_markdown)
+        if output == "all":
+            click.echo(json.dumps(result.model_dump(), indent=2))
+        elif output == "json":
+            click.echo(json.dumps(json.loads(result.extracted_content), indent=2))
+        elif output in ["markdown", "md"]:
+            click.echo(result.markdown.raw_markdown)
+        elif output in ["markdown-fit", "md-fit"]:
+            click.echo(result.markdown.fit_markdown)
             
     except Exception as e:
         raise click.ClickException(str(e))
@@ -1211,120 +1120,6 @@ def examples_cmd():
     """Show usage examples"""
     show_examples()
 
-@cli.group("config")
-def config_cmd():
-    """Manage global configuration settings
-    
-    Commands to view and update global configuration settings:
-    - list: Display all current configuration settings
-    - get: Get the value of a specific setting
-    - set: Set the value of a specific setting
-    """
-    pass
-
-@config_cmd.command("list")
-def config_list_cmd():
-    """List all configuration settings"""
-    config = get_global_config()
-    
-    table = Table(title="Crawl4AI Configuration", show_header=True, header_style="bold cyan", border_style="blue")
-    table.add_column("Setting", style="cyan")
-    table.add_column("Value", style="green")
-    table.add_column("Default", style="yellow")
-    table.add_column("Description", style="white")
-    
-    for key, setting in USER_SETTINGS.items():
-        value = config.get(key, setting["default"])
-        
-        # Handle secret values
-        display_value = value
-        if setting.get("secret", False) and value:
-            display_value = "********"
-            
-        # Handle boolean values
-        if setting["type"] == "boolean":
-            display_value = str(value).lower()
-            default_value = str(setting["default"]).lower()
-        else:
-            default_value = str(setting["default"])
-        
-        table.add_row(
-            key,
-            str(display_value),
-            default_value,
-            setting["description"]
-        )
-    
-    console.print(table)
-
-@config_cmd.command("get")
-@click.argument("key", required=True)
-def config_get_cmd(key: str):
-    """Get a specific configuration setting"""
-    config = get_global_config()
-    
-    # Normalize key to uppercase
-    key = key.upper()
-    
-    if key not in USER_SETTINGS:
-        console.print(f"[red]Error: Unknown setting '{key}'[/red]")
-        return
-    
-    value = config.get(key, USER_SETTINGS[key]["default"])
-    
-    # Handle secret values
-    display_value = value
-    if USER_SETTINGS[key].get("secret", False) and value:
-        display_value = "********"
-    
-    console.print(f"[cyan]{key}[/cyan] = [green]{display_value}[/green]")
-    console.print(f"[dim]Description: {USER_SETTINGS[key]['description']}[/dim]")
-
-@config_cmd.command("set")
-@click.argument("key", required=True)
-@click.argument("value", required=True)
-def config_set_cmd(key: str, value: str):
-    """Set a configuration setting"""
-    config = get_global_config()
-    
-    # Normalize key to uppercase
-    key = key.upper()
-    
-    if key not in USER_SETTINGS:
-        console.print(f"[red]Error: Unknown setting '{key}'[/red]")
-        console.print(f"[yellow]Available settings: {', '.join(USER_SETTINGS.keys())}[/yellow]")
-        return
-    
-    setting = USER_SETTINGS[key]
-    
-    # Type conversion and validation
-    if setting["type"] == "boolean":
-        if value.lower() in ["true", "yes", "1", "y"]:
-            typed_value = True
-        elif value.lower() in ["false", "no", "0", "n"]:
-            typed_value = False
-        else:
-            console.print(f"[red]Error: Invalid boolean value. Use 'true' or 'false'.[/red]")
-            return
-    elif setting["type"] == "string":
-        typed_value = value
-        
-        # Check if the value should be one of the allowed options
-        if "options" in setting and value not in setting["options"]:
-            console.print(f"[red]Error: Value must be one of: {', '.join(setting['options'])}[/red]")
-            return
-    
-    # Update config
-    config[key] = typed_value
-    save_global_config(config)
-    
-    # Handle secret values for display
-    display_value = typed_value
-    if setting.get("secret", False) and typed_value:
-        display_value = "********"
-        
-    console.print(f"[green]Successfully set[/green] [cyan]{key}[/cyan] = [green]{display_value}[/green]")
-
 @cli.command("profiles")
 def profiles_cmd():
     """Manage browser profiles interactively
@@ -1344,7 +1139,6 @@ def profiles_cmd():
 @click.option("--crawler-config", "-C", type=click.Path(exists=True), help="Crawler config file (YAML/JSON)")
 @click.option("--filter-config", "-f", type=click.Path(exists=True), help="Content filter config file")
 @click.option("--extraction-config", "-e", type=click.Path(exists=True), help="Extraction strategy config file")
-@click.option("--json-extract", "-j", is_flag=False, flag_value="", default=None, help="Extract structured data using LLM with optional description")
 @click.option("--schema", "-s", type=click.Path(exists=True), help="JSON schema for extraction")
 @click.option("--browser", "-b", type=str, callback=parse_key_values, help="Browser parameters as key1=value1,key2=value2")
 @click.option("--crawler", "-c", type=str, callback=parse_key_values, help="Crawler parameters as key1=value1,key2=value2")
@@ -1354,7 +1148,7 @@ def profiles_cmd():
 @click.option("--verbose", "-v", is_flag=True)
 @click.option("--profile", "-p", help="Use a specific browser profile (by name)")
 def default(url: str, example: bool, browser_config: str, crawler_config: str, filter_config: str, 
-        extraction_config: str, json_extract: str, schema: str, browser: Dict, crawler: Dict,
+        extraction_config: str, schema: str, browser: Dict, crawler: Dict,
         output: str, bypass_cache: bool, question: str, verbose: bool, profile: str):
     """Crawl4AI CLI - Web content extraction tool
 
@@ -1368,14 +1162,7 @@ def default(url: str, example: bool, browser_config: str, crawler_config: str, f
         crwl crawl      - Crawl a website with advanced options
         crwl cdp        - Launch browser with CDP debugging enabled
         crwl browser    - Manage builtin browser (start, stop, status, restart)
-        crwl config     - Manage global configuration settings
         crwl examples   - Show more usage examples
-        
-    Configuration Examples:
-        crwl config list                         - List all configuration settings
-        crwl config get DEFAULT_LLM_PROVIDER     - Show current LLM provider
-        crwl config set VERBOSE true             - Enable verbose mode globally
-        crwl config set BROWSER_HEADLESS false   - Default to visible browser
     """
 
     if example:
@@ -1396,8 +1183,7 @@ def default(url: str, example: bool, browser_config: str, crawler_config: str, f
         browser_config=browser_config,
         crawler_config=crawler_config,
         filter_config=filter_config,
-        extraction_config=extraction_config,
-        json_extract=json_extract,
+        extraction_config=extraction_config, 
         schema=schema,
         browser=browser,
         crawler=crawler,

crawl4ai/config.py

@@ -93,46 +93,3 @@ SHOW_DEPRECATION_WARNINGS = True
 SCREENSHOT_HEIGHT_TRESHOLD = 10000
 PAGE_TIMEOUT = 60000
 DOWNLOAD_PAGE_TIMEOUT = 60000
-
-# Global user settings with descriptions and default values
-USER_SETTINGS = {
-    "DEFAULT_LLM_PROVIDER": {
-        "default": "openai/gpt-4o",
-        "description": "Default LLM provider in 'company/model' format (e.g., 'openai/gpt-4o', 'anthropic/claude-3-sonnet')",
-        "type": "string"
-    },
-    "DEFAULT_LLM_PROVIDER_TOKEN": {
-        "default": "",
-        "description": "API token for the default LLM provider",
-        "type": "string",
-        "secret": True
-    },
-    "VERBOSE": {
-        "default": False,
-        "description": "Enable verbose output for all commands",
-        "type": "boolean"
-    },
-    "BROWSER_HEADLESS": {
-        "default": True,
-        "description": "Run browser in headless mode by default",
-        "type": "boolean"
-    },
-    "BROWSER_TYPE": {
-        "default": "chromium",
-        "description": "Default browser type (chromium or firefox)",
-        "type": "string",
-        "options": ["chromium", "firefox"]
-    },
-    "CACHE_MODE": {
-        "default": "bypass",
-        "description": "Default cache mode (bypass, use, or refresh)",
-        "type": "string",
-        "options": ["bypass", "use", "refresh"]
-    },
-    "USER_AGENT_MODE": {
-        "default": "default",
-        "description": "Default user agent mode (default, random, or mobile)",
-        "type": "string",
-        "options": ["default", "random", "mobile"]
-    }
-}

crawl4ai/deep_crawling/base_strategy.py

@@ -7,6 +7,7 @@ from contextvars import ContextVar
 from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
 
 
+
 class DeepCrawlDecorator:
     """Decorator that adds deep crawling capability to arun method."""
     deep_crawl_active = ContextVar("deep_crawl_active", default=False)
@@ -59,7 +60,8 @@ class DeepCrawlStrategy(ABC):
         start_url: str,
         crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
-    ) -> List[CrawlResult]:
+    # ) -> List[CrawlResult]:
+    ) -> RunManyReturn:
         """
         Batch (non-streaming) mode:
         Processes one BFS level at a time, then yields all the results.
@@ -72,7 +74,8 @@ class DeepCrawlStrategy(ABC):
         start_url: str,
         crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
-    ) -> AsyncGenerator[CrawlResult, None]:
+    # ) -> AsyncGenerator[CrawlResult, None]:
+    ) -> RunManyReturn:
         """
         Streaming mode:
         Processes one BFS level at a time and yields results immediately as they arrive.

crawl4ai/deep_crawling/bfs_strategy.py

@@ -9,7 +9,7 @@ from ..models import TraversalStats
 from .filters import FilterChain
 from .scorers import URLScorer
 from . import DeepCrawlStrategy  
-from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult
+from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
 from ..utils import normalize_url_for_deep_crawl, efficient_normalize_url_for_deep_crawl
 from math import inf as infinity
 
@@ -143,7 +143,8 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
         start_url: str,
         crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
-    ) -> List[CrawlResult]:
+    # ) -> List[CrawlResult]:
+    ) -> RunManyReturn:
         """
         Batch (non-streaming) mode:
         Processes one BFS level at a time, then yields all the results.
@@ -191,7 +192,8 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
         start_url: str,
         crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
-    ) -> AsyncGenerator[CrawlResult, None]:
+    # ) -> AsyncGenerator[CrawlResult, None]:
+    ) -> RunManyReturn:
         """
         Streaming mode:
         Processes one BFS level at a time and yields results immediately as they arrive.

crawl4ai/deep_crawling/dfs_strategy.py

@@ -3,7 +3,7 @@ from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
 
 from ..models import CrawlResult
 from .bfs_strategy import BFSDeepCrawlStrategy  # noqa
-from ..types import AsyncWebCrawler, CrawlerRunConfig
+from ..types import AsyncWebCrawler, CrawlerRunConfig, RunManyReturn
 
 class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
     """
@@ -17,7 +17,8 @@ class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
         start_url: str,
         crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
-    ) -> List[CrawlResult]:
+    # ) -> List[CrawlResult]:
+    ) -> RunManyReturn:
         """
         Batch (non-streaming) DFS mode.
         Uses a stack to traverse URLs in DFS order, aggregating CrawlResults into a list.
@@ -65,7 +66,8 @@ class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
         start_url: str,
         crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
-    ) -> AsyncGenerator[CrawlResult, None]:
+    # ) -> AsyncGenerator[CrawlResult, None]:
+    ) -> RunManyReturn:
         """
         Streaming DFS mode.
         Uses a stack to traverse URLs in DFS order and yields CrawlResults as they become available.

crawl4ai/extraction_strategy.py

@@ -5,7 +5,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 import json
 import time
 
-from .prompts import PROMPT_EXTRACT_BLOCKS, PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION, PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION, JSON_SCHEMA_BUILDER_XPATH, PROMPT_EXTRACT_INFERRED_SCHEMA
+from .prompts import PROMPT_EXTRACT_BLOCKS, PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION, PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION, JSON_SCHEMA_BUILDER_XPATH
 from .config import (
     DEFAULT_PROVIDER, CHUNK_TOKEN_THRESHOLD,
     OVERLAP_RATE,
@@ -507,7 +507,6 @@ class LLMExtractionStrategy(ExtractionStrategy):
         word_token_rate=WORD_TOKEN_RATE,
         apply_chunking=True,
         input_format: str = "markdown",
-        force_json_response=False,
         verbose=False,
         # Deprecated arguments
         provider: str = DEFAULT_PROVIDER,
@@ -528,10 +527,9 @@ class LLMExtractionStrategy(ExtractionStrategy):
             overlap_rate: Overlap between chunks.
             word_token_rate: Word to token conversion rate.
             apply_chunking: Whether to apply chunking.
-            input_format: Content format to use for extraction.
-                            Options: "markdown" (default), "html", "fit_markdown"
-            force_json_response: Whether to force a JSON response from the LLM.
             verbose: Whether to print verbose output.
+            usages: List of individual token usages.
+            total_usage: Accumulated token usage.
 
             # Deprecated arguments, will be removed very soon
             provider: The provider to use for extraction. It follows the format <provider_name>/<model_name>, e.g., "ollama/llama3.3".
@@ -547,7 +545,6 @@ class LLMExtractionStrategy(ExtractionStrategy):
         self.schema = schema
         if schema:
             self.extract_type = "schema"
-        self.force_json_response = force_json_response
         self.chunk_token_threshold = chunk_token_threshold or CHUNK_TOKEN_THRESHOLD
         self.overlap_rate = overlap_rate
         self.word_token_rate = word_token_rate
@@ -611,97 +608,64 @@ class LLMExtractionStrategy(ExtractionStrategy):
             variable_values["SCHEMA"] = json.dumps(self.schema, indent=2) # if type of self.schema is dict else self.schema
             prompt_with_variables = PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION
 
-        if self.extract_type == "schema" and not self.schema:
-            prompt_with_variables = PROMPT_EXTRACT_INFERRED_SCHEMA
-
         for variable in variable_values:
             prompt_with_variables = prompt_with_variables.replace(
                 "{" + variable + "}", variable_values[variable]
             )
 
+        response = perform_completion_with_backoff(
+            self.llm_config.provider,
+            prompt_with_variables,
+            self.llm_config.api_token,
+            base_url=self.llm_config.base_url,
+            extra_args=self.extra_args,
+        )  # , json_response=self.extract_type == "schema")
+        # Track usage
+        usage = TokenUsage(
+            completion_tokens=response.usage.completion_tokens,
+            prompt_tokens=response.usage.prompt_tokens,
+            total_tokens=response.usage.total_tokens,
+            completion_tokens_details=response.usage.completion_tokens_details.__dict__
+            if response.usage.completion_tokens_details
+            else {},
+            prompt_tokens_details=response.usage.prompt_tokens_details.__dict__
+            if response.usage.prompt_tokens_details
+            else {},
+        )
+        self.usages.append(usage)
+
+        # Update totals
+        self.total_usage.completion_tokens += usage.completion_tokens
+        self.total_usage.prompt_tokens += usage.prompt_tokens
+        self.total_usage.total_tokens += usage.total_tokens
+
         try:
-            response = perform_completion_with_backoff(
-                self.llm_config.provider,
-                prompt_with_variables,
-                self.llm_config.api_token,
-                base_url=self.llm_config.base_url,
-                json_response=self.force_json_response,
-                extra_args=self.extra_args,
-            )  # , json_response=self.extract_type == "schema")
-            # Track usage
-            usage = TokenUsage(
-                completion_tokens=response.usage.completion_tokens,
-                prompt_tokens=response.usage.prompt_tokens,
-                total_tokens=response.usage.total_tokens,
-                completion_tokens_details=response.usage.completion_tokens_details.__dict__
-                if response.usage.completion_tokens_details
-                else {},
-                prompt_tokens_details=response.usage.prompt_tokens_details.__dict__
-                if response.usage.prompt_tokens_details
-                else {},
-            )
-            self.usages.append(usage)
-
-            # Update totals
-            self.total_usage.completion_tokens += usage.completion_tokens
-            self.total_usage.prompt_tokens += usage.prompt_tokens
-            self.total_usage.total_tokens += usage.total_tokens
-
-            try:
-                response = response.choices[0].message.content
-                blocks = None
-
-                if self.force_json_response:
-                    blocks = json.loads(response)
-                    if isinstance(blocks, dict):
-                        # If it has only one key which calue is list then assign that to blocks, exampled: {"news": [..]}
-                        if len(blocks) == 1 and isinstance(list(blocks.values())[0], list):
-                            blocks = list(blocks.values())[0]
-                        else:
-                            # If it has only one key which value is not list then assign that to blocks, exampled: { "article_id": "1234", ... }
-                            blocks = [blocks]
-                    elif isinstance(blocks, list):
-                        # If it is a list then assign that to blocks
-                        blocks = blocks
-                else: 
-                    # blocks = extract_xml_data(["blocks"], response.choices[0].message.content)["blocks"]
-                    blocks = extract_xml_data(["blocks"], response)["blocks"]
-                    blocks = json.loads(blocks)
-
-                for block in blocks:
-                    block["error"] = False
-            except Exception:
-                parsed, unparsed = split_and_parse_json_objects(
-                    response
-                )
-                blocks = parsed
-                if unparsed:
-                    blocks.append(
-                        {"index": 0, "error": True, "tags": ["error"], "content": unparsed}
-                    )
-
-            if self.verbose:
-                print(
-                    "[LOG] Extracted",
-                    len(blocks),
-                    "blocks from URL:",
-                    url,
-                    "block index:",
-                    ix,
-                )
-            return blocks
-        except Exception as e:
-            if self.verbose:
-                print(f"[LOG] Error in LLM extraction: {e}")
-            # Add error information to extracted_content
-            return [
-                {
-                    "index": ix,
-                    "error": True,
-                    "tags": ["error"],
-                    "content": str(e),
-                }
+            blocks = extract_xml_data(["blocks"], response.choices[0].message.content)[
+                "blocks"
             ]
+            blocks = json.loads(blocks)
+            for block in blocks:
+                block["error"] = False
+        except Exception:
+            parsed, unparsed = split_and_parse_json_objects(
+                response.choices[0].message.content
+            )
+            blocks = parsed
+            if unparsed:
+                blocks.append(
+                    {"index": 0, "error": True, "tags": ["error"], "content": unparsed}
+                )
+
+        if self.verbose:
+            print(
+                "[LOG] Extracted",
+                len(blocks),
+                "blocks from URL:",
+                url,
+                "block index:",
+                ix,
+            )
+        return blocks
 
     def _merge(self, documents, chunk_token_threshold, overlap) -> List[str]:
         """

crawl4ai/install.py

@@ -45,8 +45,7 @@ def post_install():
     setup_home_directory()
     install_playwright()
     run_migration()
-    # TODO: Will be added in the future
-    # setup_builtin_browser()
+    setup_builtin_browser()
     logger.success("Post-installation setup completed!", tag="COMPLETE")
     
 def setup_builtin_browser():

crawl4ai/models.py

@@ -1,7 +1,5 @@
-from pydantic import BaseModel, HttpUrl, PrivateAttr, ConfigDict
+from pydantic import BaseModel, HttpUrl, PrivateAttr
 from typing import List, Dict, Optional, Callable, Awaitable, Union, Any
-from typing import AsyncGenerator
-from typing import Generic, TypeVar
 from enum import Enum
 from dataclasses import dataclass
 from .ssl_certificate import SSLCertificate
@@ -36,12 +34,34 @@ class CrawlerTaskResult:
     def success(self) -> bool:
         return self.result.success
 
+
 class CrawlStatus(Enum):
     QUEUED = "QUEUED"
     IN_PROGRESS = "IN_PROGRESS"
     COMPLETED = "COMPLETED"
     FAILED = "FAILED"
 
+
+# @dataclass
+# class CrawlStats:
+#     task_id: str
+#     url: str
+#     status: CrawlStatus
+#     start_time: Optional[datetime] = None
+#     end_time: Optional[datetime] = None
+#     memory_usage: float = 0.0
+#     peak_memory: float = 0.0
+#     error_message: str = ""
+
+#     @property
+#     def duration(self) -> str:
+#         if not self.start_time:
+#             return "0:00"
+#         end = self.end_time or datetime.now()
+#         duration = end - self.start_time
+#         return str(timedelta(seconds=int(duration.total_seconds())))
+
+
 @dataclass
 class CrawlStats:
     task_id: str
@@ -75,6 +95,7 @@ class CrawlStats:
         duration = end - start
         return str(timedelta(seconds=int(duration.total_seconds())))
 
+
 class DisplayMode(Enum):
     DETAILED = "DETAILED"
     AGGREGATED = "AGGREGATED"
@@ -91,10 +112,12 @@ class TokenUsage:
     completion_tokens_details: Optional[dict] = None
     prompt_tokens_details: Optional[dict] = None
 
+
 class UrlModel(BaseModel):
     url: HttpUrl
     forced: bool = False
 
+
 class MarkdownGenerationResult(BaseModel):
     raw_markdown: str
     markdown_with_citations: str
@@ -146,9 +169,8 @@ class CrawlResult(BaseModel):
     dispatch_result: Optional[DispatchResult] = None
     redirected_url: Optional[str] = None
 
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-    # class Config:
-    #     arbitrary_types_allowed = True
+    class Config:
+        arbitrary_types_allowed = True
 
 # NOTE: The StringCompatibleMarkdown class, custom __init__ method, property getters/setters,
 # and model_dump override all exist to support a smooth transition from markdown as a string
@@ -262,40 +284,6 @@ class StringCompatibleMarkdown(str):
     def __getattr__(self, name):
         return getattr(self._markdown_result, name)
 
-CrawlResultT = TypeVar('CrawlResultT', bound=CrawlResult)
-
-class CrawlResultContainer(Generic[CrawlResultT]):
-    def __init__(self, results: Union[CrawlResultT, List[CrawlResultT]]):
-        # Normalize to a list
-        if isinstance(results, list):
-            self._results = results
-        else:
-            self._results = [results]
-
-    def __iter__(self):
-        return iter(self._results)
-
-    def __getitem__(self, index):
-        return self._results[index]
-
-    def __len__(self):
-        return len(self._results)
-
-    def __getattr__(self, attr):
-        # Delegate attribute access to the first element.
-        if self._results:
-            return getattr(self._results[0], attr)
-        raise AttributeError(f"{self.__class__.__name__} object has no attribute '{attr}'")
-
-    def __repr__(self):
-        return f"{self.__class__.__name__}({self._results!r})"
-
-RunManyReturn = Union[
-    CrawlResultContainer[CrawlResultT],
-    AsyncGenerator[CrawlResultT, None]
-]
-
-
 # END of backward compatibility code for markdown/markdown_v2.
 # When removing this code in the future, make sure to:
 # 1. Replace the private attribute and property with a standard field
@@ -313,9 +301,9 @@ class AsyncCrawlResponse(BaseModel):
     ssl_certificate: Optional[SSLCertificate] = None
     redirected_url: Optional[str] = None
 
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-    # class Config:
-    #     arbitrary_types_allowed = True
+    class Config:
+        arbitrary_types_allowed = True
+
 
 ###############################
 # Scraping Models

crawl4ai/pipeline/__init__.py

@@ -1,6 +0,0 @@
-"""Pipeline module providing high-level crawling functionality."""
-
-from .pipeline import Pipeline, create_pipeline
-from .crawler import Crawler
-
-__all__ = ["Pipeline", "create_pipeline", "Crawler"]

crawl4ai/pipeline/crawler.py

@@ -1,406 +0,0 @@
-"""Crawler utility class for simplified crawling operations.
-
-This module provides a high-level utility class for crawling web pages
-with support for both single and multiple URL processing.
-"""
-
-import asyncio
-from typing import Dict, List, Optional, Tuple, Union, Callable
-
-from crawl4ai.models import CrawlResultContainer, CrawlResult
-from crawl4ai.pipeline.pipeline import create_pipeline
-from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
-from crawl4ai.async_logger import AsyncLogger
-from crawl4ai.browser.browser_hub import BrowserHub
-
-# Type definitions
-UrlList = List[str]
-UrlBatch = Tuple[List[str], CrawlerRunConfig]
-UrlFullBatch = Tuple[List[str], BrowserConfig, CrawlerRunConfig]
-BatchType = Union[UrlList, UrlBatch, UrlFullBatch]
-ProgressCallback = Callable[[str, str, Optional[CrawlResultContainer]], None]
-RetryStrategy = Callable[[str, int, Exception], Tuple[bool, float]]
-
-class Crawler:
-    """High-level utility class for crawling web pages.
-    
-    This class provides simplified methods for crawling both single URLs
-    and batches of URLs, with parallel processing capabilities.
-    """
-    
-    @classmethod
-    async def crawl(
-        cls, 
-        urls: Union[str, List[str]],
-        browser_config: Optional[BrowserConfig] = None,
-        crawler_config: Optional[CrawlerRunConfig] = None,
-        browser_hub: Optional[BrowserHub] = None,
-        logger: Optional[AsyncLogger] = None,
-        max_retries: int = 0,
-        retry_delay: float = 1.0,
-        use_new_loop: bool = True  # By default use a new loop for safety
-    ) -> Union[CrawlResultContainer, Dict[str, CrawlResultContainer]]:
-        """Crawl one or more URLs with the specified configurations.
-        
-        Args:
-            urls: Single URL or list of URLs to crawl
-            browser_config: Optional browser configuration
-            crawler_config: Optional crawler run configuration
-            browser_hub: Optional shared browser hub
-            logger: Optional logger instance
-            max_retries: Maximum number of retries for failed requests
-            retry_delay: Delay between retries in seconds
-            
-        Returns:
-            For a single URL: CrawlResultContainer with crawl results
-            For multiple URLs: Dict mapping URLs to their CrawlResultContainer results
-        """
-        # Handle single URL case
-        if isinstance(urls, str):
-            return await cls._crawl_single_url(
-                urls,
-                browser_config,
-                crawler_config,
-                browser_hub,
-                logger,
-                max_retries,
-                retry_delay,
-                use_new_loop
-            )
-        
-        # Handle multiple URLs case (sequential processing)
-        results = {}
-        for url in urls:
-            results[url] = await cls._crawl_single_url(
-                url,
-                browser_config,
-                crawler_config,
-                browser_hub,
-                logger,
-                max_retries,
-                retry_delay,
-                use_new_loop
-            )
-        
-        return results
-    
-    @classmethod
-    async def _crawl_single_url(
-        cls,
-        url: str,
-        browser_config: Optional[BrowserConfig] = None,
-        crawler_config: Optional[CrawlerRunConfig] = None,
-        browser_hub: Optional[BrowserHub] = None,
-        logger: Optional[AsyncLogger] = None,
-        max_retries: int = 0,
-        retry_delay: float = 1.0,
-        use_new_loop: bool = False
-    ) -> CrawlResultContainer:
-        """Internal method to crawl a single URL with retry logic."""
-        # Create a logger if none provided
-        if logger is None:
-            logger = AsyncLogger(verbose=True)
-            
-        # Create or use the provided crawler config
-        if crawler_config is None:
-            crawler_config = CrawlerRunConfig()
-            
-        attempts = 0
-        last_error = None
-        
-        # For testing purposes, each crawler gets a new event loop to avoid conflicts
-        # This is especially important in test suites where multiple tests run in sequence
-        if use_new_loop:
-            old_loop = asyncio.get_event_loop()
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
-            
-        while attempts <= max_retries:
-            try:
-                # Create a pipeline
-                pipeline_args = {}
-                if browser_config:
-                    pipeline_args["browser_config"] = browser_config
-                if browser_hub:
-                    pipeline_args["browser_hub"] = browser_hub
-                if logger:
-                    pipeline_args["logger"] = logger
-                    
-                pipeline = await create_pipeline(**pipeline_args)
-                
-                # Perform the crawl
-                result = await pipeline.crawl(url=url, config=crawler_config)
-                
-                # Close the pipeline if we created it (not using a shared hub)
-                if not browser_hub:
-                    await pipeline.close()
-                    
-                # Restore the original event loop if we created a new one
-                if use_new_loop:
-                    asyncio.set_event_loop(old_loop)
-                    loop.close()
-                    
-                return result
-                
-            except Exception as e:
-                last_error = e
-                attempts += 1
-                
-                if attempts <= max_retries:
-                    logger.warning(
-                        message="Crawl attempt {attempt} failed for {url}: {error}. Retrying in {delay}s...",
-                        tag="RETRY",
-                        params={
-                            "attempt": attempts,
-                            "url": url,
-                            "error": str(e),
-                            "delay": retry_delay
-                        }
-                    )
-                    await asyncio.sleep(retry_delay)
-                else:
-                    logger.error(
-                        message="All {attempts} crawl attempts failed for {url}: {error}",
-                        tag="FAILED",
-                        params={
-                            "attempts": attempts,
-                            "url": url,
-                            "error": str(e)
-                        }
-                    )
-        
-        # If we get here, all attempts failed
-        result = CrawlResultContainer(
-            CrawlResult(
-                url=url,
-                html="",
-                success=False,
-                error_message=f"All {attempts} crawl attempts failed: {str(last_error)}"
-            )
-        )
-        
-        # Restore the original event loop if we created a new one
-        if use_new_loop:
-            asyncio.set_event_loop(old_loop)
-            loop.close()
-            
-        return result
-    
-    @classmethod
-    async def parallel_crawl(
-        cls,
-        url_batches: Union[List[str], List[Union[UrlBatch, UrlFullBatch]]],
-        browser_config: Optional[BrowserConfig] = None,
-        crawler_config: Optional[CrawlerRunConfig] = None,
-        browser_hub: Optional[BrowserHub] = None,
-        logger: Optional[AsyncLogger] = None,
-        concurrency: int = 5,
-        max_retries: int = 0,
-        retry_delay: float = 1.0,
-        retry_strategy: Optional[RetryStrategy] = None,
-        progress_callback: Optional[ProgressCallback] = None,
-        use_new_loop: bool = True  # By default use a new loop for safety
-    ) -> Dict[str, CrawlResultContainer]:
-        """Crawl multiple URLs in parallel with concurrency control.
-        
-        Args:
-            url_batches: List of URLs or list of URL batches with configurations
-            browser_config: Default browser configuration (used if not in batch)
-            crawler_config: Default crawler configuration (used if not in batch)
-            browser_hub: Optional shared browser hub for resource efficiency
-            logger: Optional logger instance
-            concurrency: Maximum number of concurrent crawls
-            max_retries: Maximum number of retries for failed requests
-            retry_delay: Delay between retries in seconds
-            retry_strategy: Optional custom retry strategy function
-            progress_callback: Optional callback for progress reporting
-            
-        Returns:
-            Dict mapping URLs to their CrawlResultContainer results
-        """
-        # Create a logger if none provided
-        if logger is None:
-            logger = AsyncLogger(verbose=True)
-        
-        # For testing purposes, each crawler gets a new event loop to avoid conflicts
-        # This is especially important in test suites where multiple tests run in sequence
-        if use_new_loop:
-            old_loop = asyncio.get_event_loop()
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
-            
-        # Process batches to consistent format
-        processed_batches = cls._process_url_batches(
-            url_batches, browser_config, crawler_config
-        )
-        
-        # Initialize results dictionary
-        results = {}
-        
-        # Create semaphore for concurrency control
-        semaphore = asyncio.Semaphore(concurrency)
-        
-        # Create shared browser hub if not provided
-        shared_hub = browser_hub
-        if not shared_hub:
-            shared_hub = await BrowserHub.get_browser_manager(
-                config=browser_config or BrowserConfig(),
-                logger=logger,
-                max_browsers_per_config=concurrency,
-                max_pages_per_browser=1,
-                initial_pool_size=min(concurrency, 3)  # Start with a reasonable number
-            )
-        
-        try:
-            # Create worker function for each URL
-            async def process_url(url, b_config, c_config):
-                async with semaphore:
-                    # Report start if callback provided
-                    if progress_callback:
-                        await progress_callback("started", url)
-                    
-                    attempts = 0
-                    last_error = None
-                    
-                    while attempts <= max_retries:
-                        try:
-                            # Create a pipeline using the shared hub
-                            pipeline = await create_pipeline(
-                                browser_config=b_config,
-                                browser_hub=shared_hub,
-                                logger=logger
-                            )
-                            
-                            # Perform the crawl
-                            result = await pipeline.crawl(url=url, config=c_config)
-                            
-                            # Report completion if callback provided
-                            if progress_callback:
-                                await progress_callback("completed", url, result)
-                                
-                            return url, result
-                            
-                        except Exception as e:
-                            last_error = e
-                            attempts += 1
-                            
-                            # Determine if we should retry and with what delay
-                            should_retry = attempts <= max_retries
-                            delay = retry_delay
-                            
-                            # Use custom retry strategy if provided
-                            if retry_strategy and should_retry:
-                                try:
-                                    should_retry, delay = await retry_strategy(url, attempts, e)
-                                except Exception as strategy_error:
-                                    logger.error(
-                                        message="Error in retry strategy: {error}",
-                                        tag="RETRY",
-                                        params={"error": str(strategy_error)}
-                                    )
-                            
-                            if should_retry:
-                                logger.warning(
-                                    message="Crawl attempt {attempt} failed for {url}: {error}. Retrying in {delay}s...",
-                                    tag="RETRY",
-                                    params={
-                                        "attempt": attempts,
-                                        "url": url,
-                                        "error": str(e),
-                                        "delay": delay
-                                    }
-                                )
-                                await asyncio.sleep(delay)
-                            else:
-                                logger.error(
-                                    message="All {attempts} crawl attempts failed for {url}: {error}",
-                                    tag="FAILED",
-                                    params={
-                                        "attempts": attempts,
-                                        "url": url,
-                                        "error": str(e)
-                                    }
-                                )
-                                break
-                    
-                    # If we get here, all attempts failed
-                    error_result = CrawlResultContainer(
-                        CrawlResult(
-                            url=url,
-                            html="",
-                            success=False,
-                            error_message=f"All {attempts} crawl attempts failed: {str(last_error)}"
-                        )
-                    )
-                    
-                    # Report completion with error if callback provided
-                    if progress_callback:
-                        await progress_callback("completed", url, error_result)
-                        
-                    return url, error_result
-            
-            # Create tasks for all URLs
-            tasks = []
-            for urls, b_config, c_config in processed_batches:
-                for url in urls:
-                    tasks.append(process_url(url, b_config, c_config))
-            
-            # Run all tasks and collect results
-            for completed_task in asyncio.as_completed(tasks):
-                url, result = await completed_task
-                results[url] = result
-                
-            return results
-            
-        finally:
-            # Clean up the hub only if we created it
-            if not browser_hub and shared_hub:
-                await shared_hub.close()
-                
-            # Restore the original event loop if we created a new one
-            if use_new_loop:
-                asyncio.set_event_loop(old_loop)
-                loop.close()
-    
-    @classmethod
-    def _process_url_batches(
-        cls,
-        url_batches: Union[List[str], List[Union[UrlBatch, UrlFullBatch]]],
-        default_browser_config: Optional[BrowserConfig],
-        default_crawler_config: Optional[CrawlerRunConfig]
-    ) -> List[Tuple[List[str], BrowserConfig, CrawlerRunConfig]]:
-        """Process URL batches into a consistent format.
-        
-        Converts various input formats into a consistent list of
-        (urls, browser_config, crawler_config) tuples.
-        """
-        processed_batches = []
-        
-        # Handle case where input is just a list of URLs
-        if all(isinstance(item, str) for item in url_batches):
-            urls = url_batches
-            browser_config = default_browser_config or BrowserConfig()
-            crawler_config = default_crawler_config or CrawlerRunConfig()
-            processed_batches.append((urls, browser_config, crawler_config))
-            return processed_batches
-        
-        # Process each batch
-        for batch in url_batches:
-            # Handle case: (urls, crawler_config)
-            if len(batch) == 2 and isinstance(batch[1], CrawlerRunConfig):
-                urls, c_config = batch
-                b_config = default_browser_config or BrowserConfig()
-                processed_batches.append((urls, b_config, c_config))
-                
-            # Handle case: (urls, browser_config, crawler_config)
-            elif len(batch) == 3 and isinstance(batch[1], BrowserConfig) and isinstance(batch[2], CrawlerRunConfig):
-                processed_batches.append(batch)
-                
-            # Fallback for unknown formats - assume it's just a list of URLs
-            else:
-                urls = batch
-                browser_config = default_browser_config or BrowserConfig()
-                crawler_config = default_crawler_config or CrawlerRunConfig()
-                processed_batches.append((urls, browser_config, crawler_config))
-                
-        return processed_batches

crawl4ai/pipeline/middlewares.py

@@ -1,702 +0,0 @@
-import time
-import sys
-from typing import Dict, Any, List
-import json
-
-from crawl4ai.models import (
-    CrawlResult,
-    MarkdownGenerationResult,
-    ScrapingResult,
-    CrawlResultContainer,
-)
-from crawl4ai.async_database import async_db_manager
-from crawl4ai.cache_context import CacheMode, CacheContext
-from crawl4ai.utils import (
-    sanitize_input_encode,
-    InvalidCSSSelectorError,
-    fast_format_html,
-    create_box_message,
-    get_error_context,
-)
-
-
-async def initialize_context_middleware(context: Dict[str, Any]) -> int:
-    """Initialize the context with basic configuration and validation"""
-    url = context.get("url")
-    config = context.get("config")
-    
-    if not isinstance(url, str) or not url:
-        context["error_message"] = "Invalid URL, make sure the URL is a non-empty string"
-        return 0
-    
-    # Default to ENABLED if no cache mode specified
-    if config.cache_mode is None:
-        config.cache_mode = CacheMode.ENABLED
-    
-    # Create cache context
-    context["cache_context"] = CacheContext(url, config.cache_mode, False)
-    context["start_time"] = time.perf_counter()
-    
-    return 1
-
-# middlewares.py additions
-
-async def browser_hub_middleware(context: Dict[str, Any]) -> int:
-    """
-    Initialize or connect to a Browser-Hub and add it to the pipeline context.
-    
-    This middleware handles browser hub initialization for all three scenarios:
-    1. Default configuration when nothing is specified
-    2. Custom configuration when browser_config is provided
-    3. Connection to existing hub when browser_hub_connection is provided
-    
-    Args:
-        context: The pipeline context dictionary
-        
-    Returns:
-        int: 1 for success, 0 for failure
-    """
-    from crawl4ai.browser.browser_hub import BrowserHub
-    
-    try:
-        # Get configuration from context
-        browser_config = context.get("browser_config")
-        browser_hub_id = context.get("browser_hub_id")
-        browser_hub_connection = context.get("browser_hub_connection")
-        logger = context.get("logger")
-        
-        # If we already have a browser hub in context, use it
-        if context.get("browser_hub"):
-            return 1
-        
-        # Get or create Browser-Hub
-        browser_hub = await BrowserHub.get_browser_manager(
-            config=browser_config,
-            hub_id=browser_hub_id,
-            connection_info=browser_hub_connection,
-            logger=logger
-        )
-        
-        # Add to context
-        context["browser_hub"] = browser_hub
-        return 1
-    except Exception as e:
-        context["error_message"] = f"Failed to initialize browser hub: {str(e)}"
-        return 0
-
-
-async def fetch_content_middleware(context: Dict[str, Any]) -> int:
-    """
-    Fetch content from the web using the browser hub.
-    
-    This middleware uses the browser hub to get pages for crawling,
-    and properly releases them back to the pool when done.
-    
-    Args:
-        context: The pipeline context dictionary
-        
-    Returns:
-        int: 1 for success, 0 for failure
-    """
-    url = context.get("url")
-    config = context.get("config")
-    browser_hub = context.get("browser_hub")
-    logger = context.get("logger")
-    
-    # Skip if using cached result
-    if context.get("cached_result") and context.get("html"):
-        return 1
-    
-    try:
-        # Create crawler strategy without initializing its browser manager
-        from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
-        
-        crawler_strategy = AsyncPlaywrightCrawlerStrategy(
-            browser_config=browser_hub.config if browser_hub else None,
-            logger=logger
-        )
-        
-        # Replace the browser manager with our shared instance
-        crawler_strategy.browser_manager = browser_hub
-        
-        # Perform crawl without trying to initialize the browser
-        # The crawler will use the provided browser_manager to get pages
-        async_response = await crawler_strategy.crawl(url, config=config)
-        
-        # Store results in context
-        context["html"] = async_response.html
-        context["screenshot_data"] = async_response.screenshot
-        context["pdf_data"] = async_response.pdf_data
-        context["js_execution_result"] = async_response.js_execution_result
-        context["async_response"] = async_response
-        
-        return 1
-    except Exception as e:
-        context["error_message"] = f"Error fetching content: {str(e)}"
-        return 0
-
-
-async def check_cache_middleware(context: Dict[str, Any]) -> int:
-    """Check if there's a cached result and load it if available"""
-    url = context.get("url")
-    config = context.get("config")
-    cache_context = context.get("cache_context")
-    logger = context.get("logger")
-    
-    # Initialize variables
-    context["cached_result"] = None
-    context["html"] = None
-    context["extracted_content"] = None
-    context["screenshot_data"] = None
-    context["pdf_data"] = None
-    
-    # Try to get cached result if appropriate
-    if cache_context.should_read():
-        cached_result = await async_db_manager.aget_cached_url(url)
-        context["cached_result"] = cached_result
-        
-        if cached_result:
-            html = sanitize_input_encode(cached_result.html)
-            extracted_content = sanitize_input_encode(cached_result.extracted_content or "")
-            extracted_content = None if not extracted_content or extracted_content == "[]" else extracted_content
-            
-            # If screenshot is requested but its not in cache, then set cache_result to None
-            screenshot_data = cached_result.screenshot
-            pdf_data = cached_result.pdf
-            
-            if config.screenshot and not screenshot_data:
-                context["cached_result"] = None
-            
-            if config.pdf and not pdf_data:
-                context["cached_result"] = None
-            
-            context["html"] = html
-            context["extracted_content"] = extracted_content
-            context["screenshot_data"] = screenshot_data
-            context["pdf_data"] = pdf_data
-            
-            logger.url_status(
-                url=cache_context.display_url,
-                success=bool(html),
-                timing=time.perf_counter() - context["start_time"],
-                tag="FETCH",
-            )
-    
-    return 1
-
-
-async def configure_proxy_middleware(context: Dict[str, Any]) -> int:
-    """Configure proxy if a proxy rotation strategy is available"""
-    config = context.get("config")
-    logger = context.get("logger")
-    
-    # Skip if using cached result
-    if context.get("cached_result") and context.get("html"):
-        return 1
-    
-    # Update proxy configuration from rotation strategy if available
-    if config and config.proxy_rotation_strategy:
-        next_proxy = await config.proxy_rotation_strategy.get_next_proxy()
-        if next_proxy:
-            logger.info(
-                message="Switch proxy: {proxy}",
-                tag="PROXY",
-                params={"proxy": next_proxy.server},
-            )
-            config.proxy_config = next_proxy
-    
-    return 1
-
-
-async def check_robots_txt_middleware(context: Dict[str, Any]) -> int:
-    """Check if the URL is allowed by robots.txt if enabled"""
-    url = context.get("url")
-    config = context.get("config")
-    browser_config = context.get("browser_config")
-    robots_parser = context.get("robots_parser")
-    
-    # Skip if using cached result
-    if context.get("cached_result") and context.get("html"):
-        return 1
-    
-    # Check robots.txt if enabled
-    if config and config.check_robots_txt:
-        if not await robots_parser.can_fetch(url, browser_config.user_agent):
-            context["crawl_result"] = CrawlResult(
-                url=url,
-                html="",
-                success=False,
-                status_code=403,
-                error_message="Access denied by robots.txt",
-                response_headers={"X-Robots-Status": "Blocked by robots.txt"}
-            )
-            return 0
-    
-    return 1
-
-
-async def fetch_content_middleware_(context: Dict[str, Any]) -> int:
-    """Fetch content from the web using the crawler strategy"""
-    url = context.get("url")
-    config = context.get("config")
-    crawler_strategy = context.get("crawler_strategy")
-    logger = context.get("logger")
-    
-    # Skip if using cached result
-    if context.get("cached_result") and context.get("html"):
-        return 1
-    
-    try:
-        t1 = time.perf_counter()
-        
-        if config.user_agent:
-            crawler_strategy.update_user_agent(config.user_agent)
-        
-        # Call CrawlerStrategy.crawl
-        async_response = await crawler_strategy.crawl(url, config=config)
-        
-        html = sanitize_input_encode(async_response.html)
-        screenshot_data = async_response.screenshot
-        pdf_data = async_response.pdf_data
-        js_execution_result = async_response.js_execution_result
-        
-        t2 = time.perf_counter()
-        logger.url_status(
-            url=context["cache_context"].display_url,
-            success=bool(html),
-            timing=t2 - t1,
-            tag="FETCH",
-        )
-        
-        context["html"] = html
-        context["screenshot_data"] = screenshot_data
-        context["pdf_data"] = pdf_data
-        context["js_execution_result"] = js_execution_result
-        context["async_response"] = async_response
-        
-        return 1
-    except Exception as e:
-        context["error_message"] = f"Error fetching content: {str(e)}"
-        return 0
-
-
-async def scrape_content_middleware(context: Dict[str, Any]) -> int:
-    """Apply scraping strategy to extract content"""
-    url = context.get("url")
-    html = context.get("html")
-    config = context.get("config")
-    extracted_content = context.get("extracted_content")
-    logger = context.get("logger")
-    
-    # Skip if already have a crawl result
-    if context.get("crawl_result"):
-        return 1
-    
-    try:
-        _url = url if not context.get("is_raw_html", False) else "Raw HTML"
-        t1 = time.perf_counter()
-        
-        # Get scraping strategy and ensure it has a logger
-        scraping_strategy = config.scraping_strategy
-        if not scraping_strategy.logger:
-            scraping_strategy.logger = logger
-        
-        # Process HTML content
-        params = config.__dict__.copy()
-        params.pop("url", None)
-        # Add keys from kwargs to params that don't exist in params
-        kwargs = context.get("kwargs", {})
-        params.update({k: v for k, v in kwargs.items() if k not in params.keys()})
-        
-        # Scraping Strategy Execution
-        result: ScrapingResult = scraping_strategy.scrap(url, html, **params)
-        
-        if result is None:
-            raise ValueError(f"Process HTML, Failed to extract content from the website: {url}")
-        
-        # Extract results - handle both dict and ScrapingResult
-        if isinstance(result, dict):
-            cleaned_html = sanitize_input_encode(result.get("cleaned_html", ""))
-            media = result.get("media", {})
-            links = result.get("links", {})
-            metadata = result.get("metadata", {})
-        else:
-            cleaned_html = sanitize_input_encode(result.cleaned_html)
-            media = result.media.model_dump()
-            links = result.links.model_dump()
-            metadata = result.metadata
-        
-        context["cleaned_html"] = cleaned_html
-        context["media"] = media
-        context["links"] = links
-        context["metadata"] = metadata
-        
-        # Log processing completion
-        logger.info(
-            message="{url:.50}... | Time: {timing}s",
-            tag="SCRAPE",
-            params={
-                "url": _url,
-                "timing": int((time.perf_counter() - t1) * 1000) / 1000,
-            },
-        )
-        
-        return 1
-    except InvalidCSSSelectorError as e:
-        context["error_message"] = str(e)
-        return 0
-    except Exception as e:
-        context["error_message"] = f"Process HTML, Failed to extract content from the website: {url}, error: {str(e)}"
-        return 0
-
-
-async def generate_markdown_middleware(context: Dict[str, Any]) -> int:
-    """Generate markdown from cleaned HTML"""
-    url = context.get("url")
-    cleaned_html = context.get("cleaned_html")
-    config = context.get("config")
-    
-    # Skip if already have a crawl result
-    if context.get("crawl_result"):
-        return 1
-    
-    # Generate Markdown
-    markdown_generator = config.markdown_generator
-    
-    markdown_result: MarkdownGenerationResult = markdown_generator.generate_markdown(
-        cleaned_html=cleaned_html,
-        base_url=url,
-    )
-    
-    context["markdown_result"] = markdown_result
-    
-    return 1
-
-
-async def extract_structured_content_middleware(context: Dict[str, Any]) -> int:
-    """Extract structured content using extraction strategy"""
-    url = context.get("url")
-    extracted_content = context.get("extracted_content")
-    config = context.get("config")
-    markdown_result = context.get("markdown_result")
-    cleaned_html = context.get("cleaned_html")
-    logger = context.get("logger")
-    
-    # Skip if already have a crawl result or extracted content
-    if context.get("crawl_result") or bool(extracted_content):
-        return 1
-    
-    from crawl4ai.chunking_strategy import IdentityChunking
-    from crawl4ai.extraction_strategy import NoExtractionStrategy
-    
-    if config.extraction_strategy and not isinstance(config.extraction_strategy, NoExtractionStrategy):
-        t1 = time.perf_counter()
-        _url = url if not context.get("is_raw_html", False) else "Raw HTML"
-        
-        # Choose content based on input_format
-        content_format = config.extraction_strategy.input_format
-        if content_format == "fit_markdown" and not markdown_result.fit_markdown:
-            logger.warning(
-                message="Fit markdown requested but not available. Falling back to raw markdown.",
-                tag="EXTRACT",
-                params={"url": _url},
-            )
-            content_format = "markdown"
-        
-        content = {
-            "markdown": markdown_result.raw_markdown,
-            "html": context.get("html"),
-            "cleaned_html": cleaned_html,
-            "fit_markdown": markdown_result.fit_markdown,
-        }.get(content_format, markdown_result.raw_markdown)
-        
-        # Use IdentityChunking for HTML input, otherwise use provided chunking strategy
-        chunking = (
-            IdentityChunking()
-            if content_format in ["html", "cleaned_html"]
-            else config.chunking_strategy
-        )
-        sections = chunking.chunk(content)
-        extracted_content = config.extraction_strategy.run(url, sections)
-        extracted_content = json.dumps(
-            extracted_content, indent=4, default=str, ensure_ascii=False
-        )
-        
-        context["extracted_content"] = extracted_content
-        
-        # Log extraction completion
-        logger.info(
-            message="Completed for {url:.50}... | Time: {timing}s",
-            tag="EXTRACT",
-            params={"url": _url, "timing": time.perf_counter() - t1},
-        )
-    
-    return 1
-
-
-async def format_html_middleware(context: Dict[str, Any]) -> int:
-    """Format HTML if prettify is enabled"""
-    config = context.get("config")
-    cleaned_html = context.get("cleaned_html")
-    
-    # Skip if already have a crawl result
-    if context.get("crawl_result"):
-        return 1
-    
-    # Apply HTML formatting if requested
-    if config.prettiify and cleaned_html:
-        context["cleaned_html"] = fast_format_html(cleaned_html)
-    
-    return 1
-
-
-async def write_cache_middleware(context: Dict[str, Any]) -> int:
-    """Write result to cache if appropriate"""
-    cache_context = context.get("cache_context")
-    cached_result = context.get("cached_result")
-    
-    # Skip if already have a crawl result or not using cache
-    if context.get("crawl_result") or not cache_context.should_write() or bool(cached_result):
-        return 1
-    
-    # We'll create the CrawlResult in build_result_middleware and cache it there
-    # to avoid creating it twice
-    
-    return 1
-
-
-async def build_result_middleware(context: Dict[str, Any]) -> int:
-    """Build the final CrawlResult object"""
-    url = context.get("url")
-    html = context.get("html", "")
-    cache_context = context.get("cache_context")
-    cached_result = context.get("cached_result")
-    config = context.get("config")
-    logger = context.get("logger")
-    
-    # If we already have a crawl result (from an earlier middleware like robots.txt check)
-    if context.get("crawl_result"):
-        result = context["crawl_result"]
-        context["final_result"] = CrawlResultContainer(result)
-        return 1
-    
-    # If we have a cached result
-    if cached_result and html:
-        logger.success(
-            message="{url:.50}... | Status: {status} | Total: {timing}",
-            tag="COMPLETE",
-            params={
-                "url": cache_context.display_url,
-                "status": True,
-                "timing": f"{time.perf_counter() - context['start_time']:.2f}s",
-            },
-            colors={"status": "green", "timing": "yellow"},
-        )
-        
-        cached_result.success = bool(html)
-        cached_result.session_id = getattr(config, "session_id", None)
-        cached_result.redirected_url = cached_result.redirected_url or url
-        context["final_result"] = CrawlResultContainer(cached_result)
-        return 1
-    
-    # Build a new result
-    try:
-        # Get all necessary components from context
-        cleaned_html = context.get("cleaned_html", "")
-        markdown_result = context.get("markdown_result")
-        media = context.get("media", {})
-        links = context.get("links", {})
-        metadata = context.get("metadata", {})
-        screenshot_data = context.get("screenshot_data")
-        pdf_data = context.get("pdf_data")
-        extracted_content = context.get("extracted_content")
-        async_response = context.get("async_response")
-        
-        # Create the CrawlResult
-        crawl_result = CrawlResult(
-            url=url,
-            html=html,
-            cleaned_html=cleaned_html,
-            markdown=markdown_result,
-            media=media,
-            links=links,
-            metadata=metadata,
-            screenshot=screenshot_data,
-            pdf=pdf_data,
-            extracted_content=extracted_content,
-            success=bool(html),
-            error_message="",
-        )
-        
-        # Add response details if available
-        if async_response:
-            crawl_result.status_code = async_response.status_code
-            crawl_result.redirected_url = async_response.redirected_url or url
-            crawl_result.response_headers = async_response.response_headers
-            crawl_result.downloaded_files = async_response.downloaded_files
-            crawl_result.js_execution_result = context.get("js_execution_result")
-            crawl_result.ssl_certificate = async_response.ssl_certificate
-        
-        crawl_result.session_id = getattr(config, "session_id", None)
-        
-        # Log completion
-        logger.success(
-            message="{url:.50}... | Status: {status} | Total: {timing}",
-            tag="COMPLETE",
-            params={
-                "url": cache_context.display_url,
-                "status": crawl_result.success,
-                "timing": f"{time.perf_counter() - context['start_time']:.2f}s",
-            },
-            colors={
-                "status": "green" if crawl_result.success else "red",
-                "timing": "yellow",
-            },
-        )
-        
-        # Update cache if appropriate
-        if cache_context.should_write() and not bool(cached_result):
-            await async_db_manager.acache_url(crawl_result)
-        
-        context["final_result"] = CrawlResultContainer(crawl_result)
-        return 1
-    except Exception as e:
-        error_context = get_error_context(sys.exc_info())
-        
-        error_message = (
-            f"Unexpected error in build_result at line {error_context['line_no']} "
-            f"in {error_context['function']} ({error_context['filename']}):\n"
-            f"Error: {str(e)}\n\n"
-            f"Code context:\n{error_context['code_context']}"
-        )
-        
-        logger.error_status(
-            url=url,
-            error=create_box_message(error_message, type="error"),
-            tag="ERROR",
-        )
-        
-        context["final_result"] = CrawlResultContainer(
-            CrawlResult(
-                url=url, html="", success=False, error_message=error_message
-            )
-        )
-        return 1
-
-
-async def handle_error_middleware(context: Dict[str, Any]) -> Dict[str, Any]:
-    """Error handler middleware"""
-    url = context.get("url", "")
-    error_message = context.get("error_message", "Unknown error")
-    logger = context.get("logger")
-    
-    # Log the error
-    if logger:
-        logger.error_status(
-            url=url,
-            error=create_box_message(error_message, type="error"),
-            tag="ERROR",
-        )
-    
-    # Create a failure result
-    context["final_result"] = CrawlResultContainer(
-        CrawlResult(
-            url=url, html="", success=False, error_message=error_message
-        )
-    )
-    
-    return context
-
-
-# Custom middlewares as requested
-
-async def sentiment_analysis_middleware(context: Dict[str, Any]) -> int:
-    """Analyze sentiment of generated markdown using TextBlob"""
-    from textblob import TextBlob
-    
-    markdown_result = context.get("markdown_result")
-    
-    # Skip if no markdown or already failed
-    if not markdown_result or not context.get("success", True):
-        return 1
-    
-    try:
-        # Get raw markdown text
-        raw_markdown = markdown_result.raw_markdown
-        
-        # Analyze sentiment
-        blob = TextBlob(raw_markdown)
-        sentiment = blob.sentiment
-        
-        # Add sentiment to context
-        context["sentiment_analysis"] = {
-            "polarity": sentiment.polarity,  # -1.0 to 1.0 (negative to positive)
-            "subjectivity": sentiment.subjectivity,  # 0.0 to 1.0 (objective to subjective)
-            "classification": "positive" if sentiment.polarity > 0.1 else 
-                             "negative" if sentiment.polarity < -0.1 else "neutral"
-        }
-        
-        return 1
-    except Exception as e:
-        # Don't fail the pipeline on sentiment analysis failure
-        context["sentiment_analysis_error"] = str(e)
-        return 1
-
-
-async def log_timing_middleware(context: Dict[str, Any], name: str) -> int:
-    """Log timing information for a specific point in the pipeline"""
-    context[f"_timing_mark_{name}"] = time.perf_counter()
-    
-    # Calculate duration if we have a start time
-    start_key = f"_timing_start_{name}"
-    if start_key in context:
-        duration = context[f"_timing_mark_{name}"] - context[start_key]
-        context[f"_timing_duration_{name}"] = duration
-        
-        # Log the timing if we have a logger
-        logger = context.get("logger")
-        if logger:
-            logger.info(
-                message="{name} completed in {duration:.2f}s",
-                tag="TIMING",
-                params={"name": name, "duration": duration},
-            )
-    
-    return 1
-
-
-async def validate_url_middleware(context: Dict[str, Any], patterns: List[str]) -> int:
-    """Validate URL against glob patterns"""
-    import fnmatch
-    url = context.get("url", "")
-    
-    # If no patterns provided, allow all
-    if not patterns:
-        return 1
-    
-    # Check if URL matches any of the allowed patterns
-    for pattern in patterns:
-        if fnmatch.fnmatch(url, pattern):
-            return 1
-    
-    # If we get here, URL didn't match any patterns
-    context["error_message"] = f"URL '{url}' does not match any allowed patterns"
-    return 0
-
-
-# Update the default middleware list function
-def create_default_middleware_list():
-    """Return the default list of middleware functions for the pipeline."""
-    return [
-        initialize_context_middleware,
-        check_cache_middleware,
-        browser_hub_middleware,  # Add browser hub middleware before fetch_content
-        configure_proxy_middleware,
-        check_robots_txt_middleware,
-        fetch_content_middleware,
-        scrape_content_middleware,
-        generate_markdown_middleware,
-        extract_structured_content_middleware,
-        format_html_middleware,
-        build_result_middleware
-    ]

crawl4ai/pipeline/pipeline.py

@@ -1,297 +0,0 @@
-
-import time
-import asyncio
-from typing import Callable, Dict, List, Any, Optional, Awaitable, Union, TypedDict, Tuple, Coroutine
-
-from .middlewares import create_default_middleware_list, handle_error_middleware
-from crawl4ai.models import CrawlResultContainer, CrawlResult
-from crawl4ai.async_crawler_strategy import AsyncCrawlerStrategy, AsyncPlaywrightCrawlerStrategy
-from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
-from crawl4ai.async_logger import AsyncLogger
-
-
-class CrawlSpec(TypedDict, total=False):
-    """Specification for a single crawl operation in batch_crawl."""
-    url: str
-    config: Optional[CrawlerRunConfig]
-    browser_config: Optional[BrowserConfig]
-
-class BatchStatus(TypedDict, total=False):
-    """Status information for batch crawl operations."""
-    total: int
-    processed: int
-    succeeded: int
-    failed: int
-    in_progress: int
-    duration: float
-    
-class Pipeline:
-    """
-    A pipeline processor that executes a series of async middleware functions.
-    Each middleware function receives a context dictionary, updates it,
-    and returns 1 for success or 0 for failure.
-    """
-
-    def __init__(
-        self, 
-        middleware: List[Callable[[Dict[str, Any]], Awaitable[int]]] = None,
-        error_handler: Optional[Callable[[Dict[str, Any]], Awaitable[Dict[str, Any]]]] = None,
-        after_middleware_callback: Optional[Callable[[str, Dict[str, Any]], Awaitable[None]]] = None,
-        crawler_strategy: Optional[AsyncCrawlerStrategy] = None,
-        browser_config: Optional[BrowserConfig] = None,
-        logger: Optional[AsyncLogger] = None,
-        _initial_context: Optional[Dict[str, Any]] = None
-    ):
-        self.middleware = middleware or create_default_middleware_list()
-        self.error_handler = error_handler or handle_error_middleware
-        self.after_middleware_callback = after_middleware_callback
-        self.browser_config = browser_config or BrowserConfig()
-        self.logger = logger or AsyncLogger(verbose=self.browser_config.verbose)
-        self.crawler_strategy = crawler_strategy or AsyncPlaywrightCrawlerStrategy(
-            browser_config=self.browser_config,
-            logger=self.logger
-        )
-        self._initial_context = _initial_context
-        self._strategy_initialized = False
-    
-    async def _initialize_strategy__(self):
-        """Initialize the crawler strategy if not already initialized"""
-        if not self.crawler_strategy:
-            self.crawler_strategy = AsyncPlaywrightCrawlerStrategy(
-                browser_config=self.browser_config,
-                logger=self.logger
-            )
-        
-        if not self._strategy_initialized:
-            await self.crawler_strategy.__aenter__()
-            self._strategy_initialized = True
-    
-    async def _initialize_strategy(self):
-        """Initialize the crawler strategy if not already initialized"""
-        # With our new approach, we don't need to create the crawler strategy here
-        # as it will be created on-demand in fetch_content_middleware
-        
-        # Just ensure browser hub is available if needed
-        if hasattr(self, "_initial_context") and "browser_hub" not in self._initial_context:
-            # If a browser_config was provided but no browser_hub yet, 
-            # we'll let the browser_hub_middleware handle creating it
-            pass
-        
-        # Mark as initialized to prevent repeated initialization attempts
-        self._strategy_initialized = True
-
-    async def start(self):
-        """Start the crawler strategy and prepare it for use"""
-        if not self._strategy_initialized:
-            await self._initialize_strategy()
-            self._strategy_initialized = True
-        if self.crawler_strategy:
-            await self.crawler_strategy.__aenter__()
-            self._strategy_initialized = True
-        else:
-            raise ValueError("Crawler strategy is not initialized.")
-    
-    async def close(self):
-        """Close the crawler strategy and clean up resources"""
-        await self.stop()
-
-    async def stop(self):
-        """Close the crawler strategy and clean up resources"""
-        if self._strategy_initialized and self.crawler_strategy:
-            await self.crawler_strategy.__aexit__(None, None, None)
-            self._strategy_initialized = False
-    
-    async def __aenter__(self):
-        await self.start()
-        return self
-    
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        await self.close()
-    
-    async def crawl(self, url: str, config: Optional[CrawlerRunConfig] = None, **kwargs) -> CrawlResultContainer:
-        """
-        Crawl a URL and process it through the pipeline.
-        
-        Args:
-            url: The URL to crawl
-            config: Optional configuration for the crawl
-            **kwargs: Additional arguments to pass to the middleware
-            
-        Returns:
-            CrawlResultContainer: The result of the crawl
-        """
-        # Initialize strategy if needed
-        await self._initialize_strategy()
-        
-        # Create the initial context
-        context = {
-            "url": url,
-            "config": config or CrawlerRunConfig(),
-            "browser_config": self.browser_config,
-            "logger": self.logger,
-            "crawler_strategy": self.crawler_strategy,
-            "kwargs": kwargs
-        }
-        
-        # Process the pipeline
-        result_context = await self.process(context)
-        
-        # Return the final result
-        return result_context.get("final_result")
-    
-    async def process(self, initial_context: Dict[str, Any] = None) -> Dict[str, Any]:
-        """
-        Process all middleware functions with the given context.
-        
-        Args:
-            initial_context: Initial context dictionary, defaults to empty dict
-            
-        Returns:
-            Updated context dictionary after all middleware have been processed
-        """
-        context = {**self._initial_context}
-        if initial_context:
-            context.update(initial_context)
-        
-        # Record pipeline start time
-        context["_pipeline_start_time"] = time.perf_counter()
-        
-        for middleware_fn in self.middleware:
-            # Get middleware name for logging
-            middleware_name = getattr(middleware_fn, '__name__', str(middleware_fn))
-            
-            # Record start time for this middleware
-            start_time = time.perf_counter()
-            context[f"_timing_start_{middleware_name}"] = start_time
-            
-            try:
-                # Execute middleware (all middleware functions are async)
-                result = await middleware_fn(context)
-                
-                # Record completion time
-                end_time = time.perf_counter()
-                context[f"_timing_end_{middleware_name}"] = end_time
-                context[f"_timing_duration_{middleware_name}"] = end_time - start_time
-                
-                # Execute after-middleware callback if provided
-                if self.after_middleware_callback:
-                    await self.after_middleware_callback(middleware_name, context)
-                
-                # Convert boolean returns to int (True->1, False->0)
-                if isinstance(result, bool):
-                    result = 1 if result else 0
-                
-                # Handle failure
-                if result == 0:
-                    if self.error_handler:
-                        context["_error_in"] = middleware_name
-                        context["_error_at"] = time.perf_counter()
-                        return await self._handle_error(context)
-                    else:
-                        context["success"] = False
-                        context["error_message"] = f"Pipeline failed at {middleware_name}"
-                        break
-            except Exception as e:
-                # Record error information
-                context["_error_in"] = middleware_name
-                context["_error_at"] = time.perf_counter()
-                context["_exception"] = e
-                context["success"] = False
-                context["error_message"] = f"Exception in {middleware_name}: {str(e)}"
-                
-                # Call error handler if available
-                if self.error_handler:
-                    return await self._handle_error(context)
-                break
-        
-        # Record pipeline completion time
-        pipeline_end_time = time.perf_counter()
-        context["_pipeline_end_time"] = pipeline_end_time
-        context["_pipeline_duration"] = pipeline_end_time - context["_pipeline_start_time"]
-        
-        # Set success to True if not already set (no failures)
-        if "success" not in context:
-            context["success"] = True
-        
-        return context
-    
-    async def _handle_error(self, context: Dict[str, Any]) -> Dict[str, Any]:
-        """Handle errors by calling the error handler"""
-        try:
-            return await self.error_handler(context)
-        except Exception as e:
-            # If error handler fails, update context with this new error
-            context["_error_handler_exception"] = e
-            context["error_message"] = f"Error handler failed: {str(e)}"
-            return context
-
-
-
-async def create_pipeline(
-    middleware_list=None, 
-    error_handler=None,
-    after_middleware_callback=None,
-    browser_config=None,
-    browser_hub_id=None,
-    browser_hub_connection=None,
-    browser_hub=None,
-    logger=None
-) -> Pipeline:
-    """
-    Factory function to create a pipeline with Browser-Hub integration.
-    
-    Args:
-        middleware_list: List of middleware functions
-        error_handler: Error handler middleware
-        after_middleware_callback: Callback after middleware execution
-        browser_config: Configuration for the browser
-        browser_hub_id: ID for browser hub instance
-        browser_hub_connection: Connection string for existing browser hub
-        browser_hub: Existing browser hub instance to use
-        logger: Logger instance
-        
-    Returns:
-        Pipeline: Configured pipeline instance
-    """
-    # Use default middleware list if none provided
-    middleware = middleware_list or create_default_middleware_list()
-    
-    # Create the pipeline
-    pipeline = Pipeline(
-        middleware=middleware,
-        error_handler=error_handler,
-        after_middleware_callback=after_middleware_callback,
-        logger=logger
-    )
-    
-    # Set browser-related attributes in the initial context
-    pipeline._initial_context = {
-        "browser_config": browser_config,
-        "browser_hub_id": browser_hub_id,
-        "browser_hub_connection": browser_hub_connection,
-        "browser_hub": browser_hub,
-        "logger": logger
-    }
-    
-    return pipeline
-
-
-
-
-# async def create_pipeline(
-#     middleware_list: Optional[List[Callable[[Dict[str, Any]], Awaitable[int]]]] = None, 
-#     error_handler: Optional[Callable[[Dict[str, Any]], Awaitable[Dict[str, Any]]]] = None,
-#     after_middleware_callback: Optional[Callable[[str, Dict[str, Any]], Awaitable[None]]] = None,
-#     crawler_strategy = None,
-#     browser_config = None,
-#     logger = None
-# ) -> Pipeline:
-#     """Factory function to create a pipeline with the given middleware"""
-#     return Pipeline(
-#         middleware=middleware_list,
-#         error_handler=error_handler,
-#         after_middleware_callback=after_middleware_callback,
-#         crawler_strategy=crawler_strategy,
-#         browser_config=browser_config,
-#         logger=logger
-#     )

crawl4ai/pipeline/test_pipeline.py

@@ -1,109 +0,0 @@
-import asyncio
-from crawl4ai import (
-    BrowserConfig,
-    CrawlerRunConfig,
-    CacheMode,
-    DefaultMarkdownGenerator,
-    PruningContentFilter
-)
-from pipeline import Pipeline
-
-async def main():
-    # Create configuration objects
-    browser_config = BrowserConfig(headless=True, verbose=True)
-    crawler_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-        markdown_generator=DefaultMarkdownGenerator(
-            content_filter=PruningContentFilter(
-                threshold=0.48, 
-                threshold_type="fixed", 
-                min_word_threshold=0
-            )
-        ),
-    )
-    
-    # Create and use pipeline with context manager
-    async with Pipeline(browser_config=browser_config) as pipeline:
-        result = await pipeline.crawl(
-            url="https://www.example.com", 
-            config=crawler_config
-        )
-        
-        # Print the result
-        print(f"URL: {result.url}")
-        print(f"Success: {result.success}")
-        
-        if result.success:
-            print("\nMarkdown excerpt:")
-            print(result.markdown.raw_markdown[:500] + "...")
-        else:
-            print(f"Error: {result.error_message}")
-
-if __name__ == "__main__":
-    asyncio.run(main())
-
-
-class CrawlTarget:
-    def __init__(self, urls, config=None):
-        self.urls = urls
-        self.config = config
-
-    def __repr__(self):
-        return f"CrawlTarget(urls={self.urls}, config={self.config})"
-    
-
-
-
-# async def main():
-#     # Create configuration objects
-#     browser_config = BrowserConfig(headless=True, verbose=True)
-    
-#     # Define different configurations
-#     config1 = CrawlerRunConfig(
-#         cache_mode=CacheMode.BYPASS,
-#         markdown_generator=DefaultMarkdownGenerator(
-#             content_filter=PruningContentFilter(threshold=0.48)
-#         ),
-#     )
-    
-#     config2 = CrawlerRunConfig(
-#         cache_mode=CacheMode.ENABLED,
-#         screenshot=True,
-#         pdf=True
-#     )
-    
-#     # Create crawl targets
-#     targets = [
-#         CrawlTarget(
-#             urls=["https://www.example.com", "https://www.wikipedia.org"],
-#             config=config1
-#         ),
-#         CrawlTarget(
-#             urls="https://news.ycombinator.com",  
-#             config=config2
-#         ),
-#         CrawlTarget(
-#             urls=["https://github.com", "https://stackoverflow.com", "https://python.org"],
-#             config=None
-#         )
-#     ]
-    
-#     # Create and use pipeline with context manager
-#     async with Pipeline(browser_config=browser_config) as pipeline:
-#         all_results = await pipeline.crawl_batch(targets)
-        
-#         for target_key, results in all_results.items():
-#             print(f"\n===== Results for {target_key} =====")
-#             print(f"Number of URLs crawled: {len(results)}")
-            
-#             for i, result in enumerate(results):
-#                 print(f"\nURL {i+1}: {result.url}")
-#                 print(f"Success: {result.success}")
-                
-#                 if result.success:
-#                     print(f"Content length: {len(result.markdown.raw_markdown)} chars")
-#                 else:
-#                     print(f"Error: {result.error_message}")
-
-# if __name__ == "__main__":
-#     asyncio.run(main())

crawl4ai/prompts.py

@@ -203,62 +203,6 @@ Avoid Common Mistakes:
 Result
 Output the final list of JSON objects, wrapped in <blocks>...</blocks> XML tags. Make sure to close the tag properly."""
 
-PROMPT_EXTRACT_INFERRED_SCHEMA = """Here is the content from the URL:
-<url>{URL}</url>
-
-<url_content>
-{HTML}
-</url_content>
-
-Please carefully read the URL content and the user's request. Analyze the page structure and infer the most appropriate JSON schema based on the content and request.
-
-Extraction Strategy:
-1. First, determine if the page contains repetitive items (like multiple products, articles, etc.) or a single content item (like a single article or page).
-2. For repetitive items: Identify the common pattern and extract each instance as a separate JSON object in an array.
-3. For single content: Extract the key information into a comprehensive JSON object that captures the essential details.
-
-Extraction instructions:
-Return the extracted information as a list of JSON objects. For repetitive content, each object in the list should correspond to a distinct item. For single content, you may return just one detailed JSON object. Wrap the entire JSON list in <blocks>...</blocks> XML tags.
-
-Schema Design Guidelines:
-- Create meaningful property names that clearly describe the data they contain
-- Use nested objects for hierarchical information
-- Use arrays for lists of related items
-- Include all information requested by the user
-- Maintain consistency in property names and data structures
-- Only include properties that are actually present in the content
-- For dates, prefer ISO format (YYYY-MM-DD)
-- For prices or numeric values, extract them without currency symbols when possible
-
-Quality Reflection:
-Before outputting your final answer, double check that:
-1. The inferred schema makes logical sense for the type of content
-2. All requested information is included
-3. The JSON is valid and could be parsed without errors
-4. Property names are consistent and descriptive
-5. The structure is optimal for the type of data being represented
-
-Avoid Common Mistakes:
-- Do NOT add any comments using "//" or "#" in the JSON output. It causes parsing errors.
-- Make sure the JSON is properly formatted with curly braces, square brackets, and commas in the right places.
-- Do not miss closing </blocks> tag at the end of the JSON output.
-- Do not generate Python code showing how to do the task; this is your task to extract the information and return it in JSON format.
-- Ensure consistency in property names across all objects
-- Don't include empty properties or null values unless they're meaningful
-- For repetitive content, ensure all objects follow the same schema
-
-Important: If user specific instruction is provided, then stress significantly on what user is requesting and describing about the schema of end result (if any). If user is requesting to extract specific information, then focus on that and ignore the rest of the content.
-<user_request>
-{REQUEST}
-</user_request>
-
-Result:
-Output the final list of JSON objects, wrapped in <blocks>...</blocks> XML tags. Make sure to close the tag properly.
-
-DO NOT ADD ANY PRE OR POST COMMENTS. JUST RETURN THE JSON OBJECTS INSIDE <blocks>...</blocks> TAGS.
-
-CRITICAL: The content inside the <blocks> tags MUST be a direct array of JSON objects (starting with '[' and ending with ']'), not a dictionary/object containing an array. For example, use <blocks>[{...}, {...}]</blocks> instead of <blocks>{"items": [{...}, {...}]}</blocks>. This is essential for proper parsing.
-"""
 
 PROMPT_FILTER_CONTENT = """Your task is to filter and convert HTML content into clean, focused markdown that's optimized for use with LLMs and information retrieval systems.
 

crawl4ai/utils.py

@@ -1551,7 +1551,7 @@ def extract_xml_tags(string):
     return list(set(tags))
 
 
-def extract_xml_data_legacy(tags, string):
+def extract_xml_data(tags, string):
     """
     Extract data for specified XML tags from a string.
 
@@ -1580,38 +1580,6 @@ def extract_xml_data_legacy(tags, string):
 
     return data
 
-def extract_xml_data(tags, string):
-    """
-    Extract data for specified XML tags from a string, returning the longest content for each tag.
-
-    How it works:
-    1. Finds all occurrences of each tag in the string using regex.
-    2. For each tag, selects the occurrence with the longest content.
-    3. Returns a dictionary of tag-content pairs.
-
-    Args:
-        tags (List[str]): The list of XML tags to extract.
-        string (str): The input string containing XML data.
-
-    Returns:
-        Dict[str, str]: A dictionary with tag names as keys and longest extracted content as values.
-    """
-
-    data = {}
-
-    for tag in tags:
-        pattern = f"<{tag}>(.*?)</{tag}>"
-        matches = re.findall(pattern, string, re.DOTALL)
-        
-        if matches:
-            # Find the longest content for this tag
-            longest_content = max(matches, key=len).strip()
-            data[tag] = longest_content
-        else:
-            data[tag] = ""
-
-    return data
-
 
 def perform_completion_with_backoff(
     provider,
@@ -1680,19 +1648,6 @@ def perform_completion_with_backoff(
                         "content": ["Rate limit error. Please try again later."],
                     }
                 ]
-        except Exception as e:
-            raise e  # Raise any other exceptions immediately
-            # print("Error during completion request:", str(e))
-            # error_message = e.message
-            # return [
-            #     {
-            #         "index": 0,
-            #         "tags": ["error"],
-            #         "content": [
-            #             f"Error during LLM completion request. {error_message}"
-            #         ],
-            #     }
-            # ]
 
 
 def extract_blocks(url, html, provider=DEFAULT_PROVIDER, api_token=None, base_url=None):

docs/examples/crypto_analysis_example.py

@@ -18,20 +18,11 @@ Key Features:
 
 import asyncio
 import pandas as pd
-import numpy as np
-import re
 import plotly.express as px
-from crawl4ai import (
-    AsyncWebCrawler,
-    BrowserConfig,
-    CrawlerRunConfig,
-    CacheMode,
-    LXMLWebScrapingStrategy,
-)
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LXMLWebScrapingStrategy
 from crawl4ai import CrawlResult
 from typing import List
-
-__current_dir__ = __file__.rsplit("/", 1)[0]
+from IPython.display import HTML
 
 class CryptoAlphaGenerator:
     """
@@ -40,319 +31,134 @@ class CryptoAlphaGenerator:
     - Liquidity scores
     - Momentum-risk ratios
     - Machine learning-inspired trading signals
-
+    
     Methods:
     analyze_tables(): Process raw tables into trading insights
     create_visuals(): Generate institutional-grade visualizations
     generate_insights(): Create plain English trading recommendations
     """
-
+    
     def clean_data(self, df: pd.DataFrame) -> pd.DataFrame:
         """
-        Convert crypto market data to machine-readable format.
-        Handles currency symbols, units (B=Billions), and percentage values.
+        Convert crypto market data to machine-readable format
+        Handles currency symbols, units (B=Billions), and percentage values
         """
-        # Make a copy to avoid SettingWithCopyWarning
-        df = df.copy()
-        
-        # Clean Price column (handle currency symbols)
-        df["Price"] = df["Price"].astype(str).str.replace("[^\d.]", "", regex=True).astype(float)
-        
-        # Handle Market Cap and Volume, considering both Billions and Trillions
-        def convert_large_numbers(value):
-            if pd.isna(value):
-                return float('nan')
-            value = str(value)
-            multiplier = 1
-            if 'B' in value:
-                multiplier = 1e9
-            elif 'T' in value:
-                multiplier = 1e12
-            # Handle cases where the value might already be numeric
-            cleaned_value = re.sub(r"[^\d.]", "", value)
-            return float(cleaned_value) * multiplier if cleaned_value else float('nan')
-        
-        df["Market Cap"] = df["Market Cap"].apply(convert_large_numbers)
-        df["Volume(24h)"] = df["Volume(24h)"].apply(convert_large_numbers)
+        # Clean numeric columns
+        df['Price'] = df['Price'].str.replace('[^\d.]', '', regex=True).astype(float)
+        df['Market Cap'] = df['Market Cap'].str.extract(r'\$([\d.]+)B')[0].astype(float) * 1e9
+        df['Volume(24h)'] = df['Volume(24h)'].str.extract(r'\$([\d.]+)B')[0].astype(float) * 1e9
         
         # Convert percentages to decimal values
-        for col in ["1h %", "24h %", "7d %"]:
-            if col in df.columns:
-                # First ensure it's string, then clean
-                df[col] = (
-                    df[col].astype(str)
-                    .str.replace("%", "")
-                    .str.replace(",", ".")
-                    .replace("nan", np.nan)
-                )
-                df[col] = pd.to_numeric(df[col], errors='coerce') / 100
-        
+        for col in ['1h %', '24h %', '7d %']:
+            df[col] = df[col].str.replace('%', '').astype(float) / 100
+            
         return df
 
     def calculate_metrics(self, df: pd.DataFrame) -> pd.DataFrame:
         """
         Compute advanced trading metrics used by quantitative funds:
-
+        
         1. Volume/Market Cap Ratio - Measures liquidity efficiency
-           (High ratio = Underestimated attention, and small-cap = higher growth potential)
-
-        2. Volatility Score - Risk-adjusted momentum potential - Shows how stable is the trend
+           (High ratio = Underestimated attention)
+           
+        2. Volatility Score - Risk-adjusted momentum potential
            (STD of 1h/24h/7d returns)
-
-        3. Momentum Score - Weighted average of returns - Shows how strong is the trend
+           
+        3. Momentum Score - Weighted average of returns
            (1h:30% + 24h:50% + 7d:20%)
-
+           
         4. Volume Anomaly - 3σ deviation detection
-           (Flags potential insider activity) - Unusual trading activity – Flags coins with volume spikes (potential insider buying or news).
+           (Flags potential insider activity)
         """
         # Liquidity Metrics
-        df["Volume/Market Cap Ratio"] = df["Volume(24h)"] / df["Market Cap"]
-
+        df['Volume/Market Cap Ratio'] = df['Volume(24h)'] / df['Market Cap']
+        
         # Risk Metrics
-        df["Volatility Score"] = df[["1h %", "24h %", "7d %"]].std(axis=1)
-
+        df['Volatility Score'] = df[['1h %','24h %','7d %']].std(axis=1)
+        
         # Momentum Metrics
-        df["Momentum Score"] = df["1h %"] * 0.3 + df["24h %"] * 0.5 + df["7d %"] * 0.2
-
+        df['Momentum Score'] = (df['1h %']*0.3 + df['24h %']*0.5 + df['7d %']*0.2)
+        
         # Anomaly Detection
-        median_vol = df["Volume(24h)"].median()
-        df["Volume Anomaly"] = df["Volume(24h)"] > 3 * median_vol
-
+        median_vol = df['Volume(24h)'].median()
+        df['Volume Anomaly'] = df['Volume(24h)'] > 3 * median_vol
+        
         # Value Flags
-        # Undervalued Flag - Low market cap and high momentum
-        # (High growth potential and low attention)
-        df["Undervalued Flag"] = (df["Market Cap"] < 1e9) & (
-            df["Momentum Score"] > 0.05
-        )
-        # Liquid Giant Flag - High volume/market cap ratio and large market cap
-        # (High liquidity and large market cap = institutional interest)
-        df["Liquid Giant"] = (df["Volume/Market Cap Ratio"] > 0.15) & (
-            df["Market Cap"] > 1e9
-        )
-
+        df['Undervalued Flag'] = (df['Market Cap'] < 1e9) & (df['Momentum Score'] > 0.05)
+        df['Liquid Giant'] = (df['Volume/Market Cap Ratio'] > 0.15) & (df['Market Cap'] > 1e9)
+        
         return df
 
-    def generate_insights_simple(self, df: pd.DataFrame) -> str:
+    def create_visuals(self, df: pd.DataFrame) -> dict:
         """
-        Generates an ultra-actionable crypto trading report with:
-        - Risk-tiered opportunities (High/Medium/Low)
-        - Concrete examples for each trade type
-        - Entry/exit strategies spelled out
-        - Visual cues for quick scanning
+        Generate three institutional-grade visualizations:
+        
+        1. 3D Market Map - X:Size, Y:Liquidity, Z:Momentum
+        2. Liquidity Tree - Color:Volume Efficiency
+        3. Momentum Leaderboard - Top sustainable movers
         """
-        report = [
-            "🚀 **CRYPTO TRADING CHEAT SHEET** 🚀",
-            "*Based on quantitative signals + hedge fund tactics*",
-            "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-        ]
-
-        # 1. HIGH-RISK: Undervalued Small-Caps (Momentum Plays)
-        high_risk = df[df["Undervalued Flag"]].sort_values("Momentum Score", ascending=False)
-        if not high_risk.empty:
-            example_coin = high_risk.iloc[0]
-            report.extend([
-                "\n🔥 **HIGH-RISK: Rocket Fuel Small-Caps**",
-                f"*Example Trade:* {example_coin['Name']} (Price: ${example_coin['Price']:.6f})",
-                "📊 *Why?* Tiny market cap (<$1B) but STRONG momentum (+{:.0f}% last week)".format(example_coin['7d %']*100),
-                "🎯 *Strategy:*",
-                "1. Wait for 5-10% dip from recent high (${:.6f} → Buy under ${:.6f})".format(
-                    example_coin['Price'] / (1 - example_coin['24h %']),  # Approx recent high
-                    example_coin['Price'] * 0.95
-                ),
-                "2. Set stop-loss at -10% (${:.6f})".format(example_coin['Price'] * 0.90),
-                "3. Take profit at +20% (${:.6f})".format(example_coin['Price'] * 1.20),
-                "⚠️ *Risk Warning:* These can drop 30% fast! Never bet more than 5% of your portfolio.",
-                "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-            ])
-
-        # 2. MEDIUM-RISK: Liquid Giants (Swing Trades)
-        medium_risk = df[df["Liquid Giant"]].sort_values("Volume/Market Cap Ratio", ascending=False)
-        if not medium_risk.empty:
-            example_coin = medium_risk.iloc[0]
-            report.extend([
-                "\n💎 **MEDIUM-RISK: Liquid Giants (Safe Swing Trades)**",
-                f"*Example Trade:* {example_coin['Name']} (Market Cap: ${example_coin['Market Cap']/1e9:.1f}B)",
-                "📊 *Why?* Huge volume (${:.1f}M/day) makes it easy to enter/exit".format(example_coin['Volume(24h)']/1e6),
-                "🎯 *Strategy:*",
-                "1. Buy when 24h volume > 15% of market cap (Current: {:.0f}%)".format(example_coin['Volume/Market Cap Ratio']*100),
-                "2. Hold 1-4 weeks (Big coins trend longer)",
-                "3. Exit when momentum drops below 5% (Current: {:.0f}%)".format(example_coin['Momentum Score']*100),
-                "📉 *Pro Tip:* Watch Bitcoin's trend - if BTC drops 5%, these usually follow.",
-                "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-            ])
-
-        # 3. LOW-RISK: Stable Momentum (DCA Targets)
-        low_risk = df[
-            (df["Momentum Score"] > 0.05) & 
-            (df["Volatility Score"] < 0.03)
-        ].sort_values("Market Cap", ascending=False)
-        if not low_risk.empty:
-            example_coin = low_risk.iloc[0]
-            report.extend([
-                "\n🛡️ **LOW-RISK: Steady Climbers (DCA & Forget)**",
-                f"*Example Trade:* {example_coin['Name']} (Volatility: {example_coin['Volatility Score']:.2f}/5)",
-                "📊 *Why?* Rises steadily (+{:.0f}%/week) with LOW drama".format(example_coin['7d %']*100),
-                "🎯 *Strategy:*",
-                "1. Buy small amounts every Tuesday/Friday (DCA)",
-                "2. Hold for 3+ months (Compound gains work best here)",
-                "3. Sell 10% at every +25% milestone",
-                "💰 *Best For:* Long-term investors who hate sleepless nights",
-                "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-            ])
-
-        # Volume Spike Alerts
-        anomalies = df[df["Volume Anomaly"]].sort_values("Volume(24h)", ascending=False)
-        if not anomalies.empty:
-            example_coin = anomalies.iloc[0]
-            report.extend([
-                "\n🚨 **Volume Spike Alert (Possible News/Whale Action)**",
-                f"*Coin:* {example_coin['Name']} (Volume: ${example_coin['Volume(24h)']/1e6:.1f}M, usual: ${example_coin['Volume(24h)']/3/1e6:.1f}M)",
-                "🔍 *Check:* Twitter/CoinGecko for news before trading",
-                "⚡ *If no news:* Could be insider buying - watch price action:",
-                "- Break above today's high → Buy with tight stop-loss",
-                "- Fade back down → Avoid (may be a fakeout)"
-            ])
-
-        # Pro Tip Footer
-        report.append("\n✨ *Pro Tip:* Bookmark this report & check back in 24h to see if signals held up.")
-
-        return "\n".join(report)
+        # 3D Market Overview
+        fig1 = px.scatter_3d(
+            df, 
+            x='Market Cap', 
+            y='Volume/Market Cap Ratio',
+            z='Momentum Score',
+            size='Volatility Score',
+            color='Volume Anomaly',
+            hover_name='Name',
+            title='Smart Money Market Map: Spot Overlooked Opportunities',
+            labels={'Market Cap': 'Size (Log $)', 'Volume/Market Cap Ratio': 'Liquidity Power'},
+            log_x=True,
+            template='plotly_dark'
+        )
+        
+        # Liquidity Efficiency Tree
+        fig2 = px.treemap(
+            df,
+            path=['Name'], 
+            values='Market Cap',
+            color='Volume/Market Cap Ratio',
+            hover_data=['Momentum Score'],
+            title='Liquidity Forest: Green = High Trading Efficiency',
+            color_continuous_scale='RdYlGn'
+        )
+        
+        # Momentum Leaders
+        fig3 = px.bar(
+            df.sort_values('Momentum Score', ascending=False).head(10),
+            x='Name', 
+            y='Momentum Score',
+            color='Volatility Score',
+            title='Sustainable Momentum Leaders (Low Volatility + High Growth)',
+            text='7d %',
+            template='plotly_dark'
+        )
+        
+        return {'market_map': fig1, 'liquidity_tree': fig2, 'momentum_leaders': fig3}
 
     def generate_insights(self, df: pd.DataFrame) -> str:
         """
-        Generates a tactical trading report with:
-        - Top 3 trades per risk level (High/Medium/Low)
-        - Auto-calculated entry/exit prices
-        - BTC chart toggle tip
+        Create plain English trading insights explaining:
+        - Volume spikes and their implications
+        - Risk-reward ratios of top movers
+        - Liquidity warnings for large positions
         """
-        # Filter top candidates for each risk level
-        high_risk = (
-            df[df["Undervalued Flag"]]
-            .sort_values("Momentum Score", ascending=False)
-            .head(3)
-        )
-        medium_risk = (
-            df[df["Liquid Giant"]]
-            .sort_values("Volume/Market Cap Ratio", ascending=False)
-            .head(3)
-        )
-        low_risk = (
-            df[(df["Momentum Score"] > 0.05) & (df["Volatility Score"] < 0.03)]
-            .sort_values("Momentum Score", ascending=False)
-            .head(3)
-        )
-
-        report = ["# 🎯 Crypto Trading Tactical Report (Top 3 Per Risk Tier)"]
+        top_coin = df.sort_values('Momentum Score', ascending=False).iloc[0]
+        anomaly_coins = df[df['Volume Anomaly']].sort_values('Volume(24h)', ascending=False)
         
-        # 1. High-Risk Trades (Small-Cap Momentum)
-        if not high_risk.empty:
-            report.append("\n## 🔥 HIGH RISK: Small-Cap Rockets (5-50% Potential)")
-            for i, coin in high_risk.iterrows():
-                current_price = coin["Price"]
-                entry = current_price * 0.95  # -5% dip
-                stop_loss = current_price * 0.90  # -10%
-                take_profit = current_price * 1.20  # +20%
-                
-                report.append(
-                    f"\n### {coin['Name']} (Momentum: {coin['Momentum Score']:.1%})"
-                    f"\n- **Current Price:** ${current_price:.4f}"
-                    f"\n- **Entry:** < ${entry:.4f} (Wait for pullback)"
-                    f"\n- **Stop-Loss:** ${stop_loss:.4f} (-10%)"
-                    f"\n- **Target:** ${take_profit:.4f} (+20%)"
-                    f"\n- **Risk/Reward:** 1:2"
-                    f"\n- **Watch:** Volume spikes above {coin['Volume(24h)']/1e6:.1f}M"
-                )
-
-        # 2. Medium-Risk Trades (Liquid Giants)
-        if not medium_risk.empty:
-            report.append("\n## 💎 MEDIUM RISK: Liquid Swing Trades (10-30% Potential)")
-            for i, coin in medium_risk.iterrows():
-                current_price = coin["Price"]
-                entry = current_price * 0.98  # -2% dip
-                stop_loss = current_price * 0.94  # -6%
-                take_profit = current_price * 1.15  # +15%
-                
-                report.append(
-                    f"\n### {coin['Name']} (Liquidity Score: {coin['Volume/Market Cap Ratio']:.1%})"
-                    f"\n- **Current Price:** ${current_price:.2f}"
-                    f"\n- **Entry:** < ${entry:.2f} (Buy slight dips)"
-                    f"\n- **Stop-Loss:** ${stop_loss:.2f} (-6%)"
-                    f"\n- **Target:** ${take_profit:.2f} (+15%)"
-                    f"\n- **Hold Time:** 1-3 weeks"
-                    f"\n- **Key Metric:** Volume/Cap > 15%"
-                )
-
-        # 3. Low-Risk Trades (Stable Momentum)
-        if not low_risk.empty:
-            report.append("\n## 🛡️ LOW RISK: Steady Gainers (5-15% Potential)")
-            for i, coin in low_risk.iterrows():
-                current_price = coin["Price"]
-                entry = current_price * 0.99  # -1% dip
-                stop_loss = current_price * 0.97  # -3%
-                take_profit = current_price * 1.10  # +10%
-                
-                report.append(
-                    f"\n### {coin['Name']} (Stability Score: {1/coin['Volatility Score']:.1f}x)"
-                    f"\n- **Current Price:** ${current_price:.2f}"
-                    f"\n- **Entry:** < ${entry:.2f} (Safe zone)"
-                    f"\n- **Stop-Loss:** ${stop_loss:.2f} (-3%)"
-                    f"\n- **Target:** ${take_profit:.2f} (+10%)"
-                    f"\n- **DCA Suggestion:** 3 buys over 72 hours"
-                )
-
-        # Volume Anomaly Alert
-        anomalies = df[df["Volume Anomaly"]].sort_values("Volume(24h)", ascending=False).head(2)
-        if not anomalies.empty:
-            report.append("\n⚠️ **Volume Spike Alerts**")
-            for i, coin in anomalies.iterrows():
-                report.append(
-                    f"- {coin['Name']}: Volume {coin['Volume(24h)']/1e6:.1f}M "
-                    f"(3x normal) | Price moved: {coin['24h %']:.1%}"
-                )
-
-        # Pro Tip
-        report.append(
-            "\n📊 **Chart Hack:** Hide BTC in visuals:\n"
-            "```python\n"
-            "# For 3D Map:\n"
-            "fig.update_traces(visible=False, selector={'name':'Bitcoin'})\n"
-            "# For Treemap:\n"
-            "df = df[df['Name'] != 'Bitcoin']\n"
-            "```"
-        )
-
-        return "\n".join(report)
-
-    def create_visuals(self, df: pd.DataFrame) -> dict:
-        """Enhanced visuals with BTC toggle support"""
-        # 3D Market Map (with BTC toggle hint)
-        fig1 = px.scatter_3d(
-            df,
-            x="Market Cap",
-            y="Volume/Market Cap Ratio",
-            z="Momentum Score",
-            color="Name",  # Color by name to allow toggling
-            hover_name="Name",
-            title="Market Map (Toggle BTC in legend to focus on alts)",
-            log_x=True
-        )
-        fig1.update_traces(
-            marker=dict(size=df["Volatility Score"]*100 + 5)  # Dynamic sizing
-        )
+        report = f"""
+        🚀 Top Alpha Opportunity: {top_coin['Name']}
+        - Momentum Score: {top_coin['Momentum Score']:.2%} (Top 1%)
+        - Risk-Reward Ratio: {top_coin['Momentum Score']/top_coin['Volatility Score']:.1f}
+        - Liquidity Warning: {'✅ Safe' if top_coin['Liquid Giant'] else '⚠️ Thin Markets'}
         
-        # Liquidity Tree (exclude BTC if too dominant)
-        if df[df["Name"] == "BitcoinBTC"]["Market Cap"].values[0] > df["Market Cap"].median() * 10:
-            df = df[df["Name"] != "BitcoinBTC"]
+        🔥 Volume Spikes Detected ({len(anomaly_coins)} coins):
+        {anomaly_coins[['Name', 'Volume(24h)']].head(3).to_markdown(index=False)}
         
-        fig2 = px.treemap(
-            df,
-            path=["Name"],
-            values="Market Cap",
-            color="Volume/Market Cap Ratio",
-            title="Liquidity Tree (BTC auto-removed if dominant)"
-        )
-        
-        return {"market_map": fig1, "liquidity_tree": fig2}
+        💡 Smart Money Tip: Coins with Volume/Cap > 15% and Momentum > 5% 
+        historically outperform by 22% weekly returns.
+        """
+        return report
 
 async def main():
     """
@@ -365,79 +171,60 @@ async def main():
     """
     # Configure browser with anti-detection features
     browser_config = BrowserConfig(
-        headless=False,
+        headless=True,
+        stealth=True,
+        block_resources=["image", "media"]
     )
-
+    
     # Initialize crawler with smart table detection
     crawler = AsyncWebCrawler(config=browser_config)
     await crawler.start()
-
+    
     try:
         # Set up scraping parameters
         crawl_config = CrawlerRunConfig(
             cache_mode=CacheMode.BYPASS,
-            table_score_threshold=8,  # Strict table detection
-            keep_data_attributes=True,
-            scraping_strategy=LXMLWebScrapingStrategy(),
-            scan_full_page=True,
-            scroll_delay=0.2,
+            scraping_strategy=LXMLWebScrapingStrategy(
+                table_score_threshold=8,  # Strict table detection
+                keep_data_attributes=True
+            )
         )
-
-        # # Execute market data extraction
-        # results: List[CrawlResult] = await crawler.arun(
-        #     url="https://coinmarketcap.com/?page=1", config=crawl_config
-        # )
-
-        # # Process results
-        # raw_df = pd.DataFrame()
-        # for result in results:
-        #     if result.success and result.media["tables"]:
-        #         # Extract primary market table
-        #         # DataFrame
-        #         raw_df = pd.DataFrame(
-        #             result.media["tables"][0]["rows"],
-        #             columns=result.media["tables"][0]["headers"],
-        #         )
-        #         break
-
-
-        # This is for debugging only
-        # ////// Remove this in production from here..
-        # Save raw data for debugging
-        # raw_df.to_csv(f"{__current_dir__}/tmp/raw_crypto_data.csv", index=False)
-        # print("🔍 Raw data saved to 'raw_crypto_data.csv'")
-
-        # Read from file for debugging
-        raw_df = pd.read_csv(f"{__current_dir__}/tmp/raw_crypto_data.csv")
-        # ////// ..to here
-
-        # Select top 20
-        raw_df = raw_df.head(50)
-        # Remove "Buy" from name
-        raw_df["Name"] = raw_df["Name"].str.replace("Buy", "")
-
-        # Initialize analysis engine
-        analyzer = CryptoAlphaGenerator()
-        clean_df = analyzer.clean_data(raw_df)
-        analyzed_df = analyzer.calculate_metrics(clean_df)
-
-        # Generate outputs
-        visuals = analyzer.create_visuals(analyzed_df)
-        insights = analyzer.generate_insights(analyzed_df)
-
-        # Save visualizations
-        visuals["market_map"].write_html(f"{__current_dir__}/tmp/market_map.html")
-        visuals["liquidity_tree"].write_html(f"{__current_dir__}/tmp/liquidity_tree.html")
-
-        # Display results
-        print("🔑 Key Trading Insights:")
-        print(insights)
-        print("\n📊 Open 'market_map.html' for interactive analysis")
-        print("\n📊 Open 'liquidity_tree.html' for interactive analysis")
+        
+        # Execute market data extraction
+        results: List[CrawlResult] = await crawler.arun(
+            url='https://coinmarketcap.com/?page=1',
+            config=crawl_config
+        )
+        
+        # Process results
+        for result in results:
+            if result.success and result.media['tables']:
+                # Extract primary market table
+                raw_df = pd.DataFrame(
+                    result.media['tables'][0]['rows'],
+                    columns=result.media['tables'][0]['headers']
+                )
+                
+                # Initialize analysis engine
+                analyzer = CryptoAlphaGenerator()
+                clean_df = analyzer.clean_data(raw_df)
+                analyzed_df = analyzer.calculate_metrics(clean_df)
+                
+                # Generate outputs
+                visuals = analyzer.create_visuals(analyzed_df)
+                insights = analyzer.generate_insights(analyzed_df)
+                
+                # Save visualizations
+                visuals['market_map'].write_html("market_map.html")
+                visuals['liquidity_tree'].write_html("liquidity_tree.html")
+                
+                # Display results
+                print("🔑 Key Trading Insights:")
+                print(insights)
+                print("\n📊 Open 'market_map.html' for interactive analysis")
 
     finally:
         await crawler.close()
 
-
 if __name__ == "__main__":
-    asyncio.run(main())
+    asyncio.run(main())

docs/examples/quickstart_async.py

@@ -0,0 +1,675 @@
+import os, sys
+
+from crawl4ai import LLMConfig
+
+# append parent directory to system path
+sys.path.append(
+    os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+)
+os.environ["FIRECRAWL_API_KEY"] = "fc-84b370ccfad44beabc686b38f1769692"
+
+import asyncio
+# import nest_asyncio
+# nest_asyncio.apply()
+
+import time
+import json
+import os
+import re
+from typing import Dict, List
+from bs4 import BeautifulSoup
+from pydantic import BaseModel, Field
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.extraction_strategy import (
+    JsonCssExtractionStrategy,
+    LLMExtractionStrategy,
+)
+
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+print("Crawl4AI: Advanced Web Crawling and Data Extraction")
+print("GitHub Repository: https://github.com/unclecode/crawl4ai")
+print("Twitter: @unclecode")
+print("Website: https://crawl4ai.com")
+
+
+async def simple_crawl():
+    print("\n--- Basic Usage ---")
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business", cache_mode=CacheMode.BYPASS
+        )
+        print(result.markdown[:500])  # Print first 500 characters
+
+
+async def simple_example_with_running_js_code():
+    print("\n--- Executing JavaScript and Using CSS Selectors ---")
+    # New code to handle the wait_for parameter
+    wait_for = """() => {
+        return Array.from(document.querySelectorAll('article.tease-card')).length > 10;
+    }"""
+
+    # wait_for can be also just a css selector
+    # wait_for = "article.tease-card:nth-child(10)"
+
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        js_code = [
+            "const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"
+        ]
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+            js_code=js_code,
+            # wait_for=wait_for,
+            cache_mode=CacheMode.BYPASS,
+        )
+        print(result.markdown[:500])  # Print first 500 characters
+
+
+async def simple_example_with_css_selector():
+    print("\n--- Using CSS Selectors ---")
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+            css_selector=".wide-tease-item__description",
+            cache_mode=CacheMode.BYPASS,
+        )
+        print(result.markdown[:500])  # Print first 500 characters
+
+
+async def use_proxy():
+    print("\n--- Using a Proxy ---")
+    print(
+        "Note: Replace 'http://your-proxy-url:port' with a working proxy to run this example."
+    )
+    # Uncomment and modify the following lines to use a proxy
+    async with AsyncWebCrawler(
+        verbose=True, proxy="http://your-proxy-url:port"
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business", cache_mode=CacheMode.BYPASS
+        )
+        if result.success:
+            print(result.markdown[:500])  # Print first 500 characters
+
+
+async def capture_and_save_screenshot(url: str, output_path: str):
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        result = await crawler.arun(
+            url=url, screenshot=True, cache_mode=CacheMode.BYPASS
+        )
+
+        if result.success and result.screenshot:
+            import base64
+
+            # Decode the base64 screenshot data
+            screenshot_data = base64.b64decode(result.screenshot)
+
+            # Save the screenshot as a JPEG file
+            with open(output_path, "wb") as f:
+                f.write(screenshot_data)
+
+            print(f"Screenshot saved successfully to {output_path}")
+        else:
+            print("Failed to capture screenshot")
+
+
+class OpenAIModelFee(BaseModel):
+    model_name: str = Field(..., description="Name of the OpenAI model.")
+    input_fee: str = Field(..., description="Fee for input token for the OpenAI model.")
+    output_fee: str = Field(
+        ..., description="Fee for output token for the OpenAI model."
+    )
+
+
+async def extract_structured_data_using_llm(
+    provider: str, api_token: str = None, extra_headers: Dict[str, str] = None
+):
+    print(f"\n--- Extracting Structured Data with {provider} ---")
+
+    if api_token is None and provider != "ollama":
+        print(f"API token is required for {provider}. Skipping this example.")
+        return
+
+    # extra_args = {}
+    extra_args = {
+        "temperature": 0,
+        "top_p": 0.9,
+        "max_tokens": 2000,
+        # any other supported parameters for litellm
+    }
+    if extra_headers:
+        extra_args["extra_headers"] = extra_headers
+
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        result = await crawler.arun(
+            url="https://openai.com/api/pricing/",
+            word_count_threshold=1,
+            extraction_strategy=LLMExtractionStrategy(
+                llm_config=LLMConfig(provider=provider,api_token=api_token),
+                schema=OpenAIModelFee.model_json_schema(),
+                extraction_type="schema",
+                instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. 
+                Do not miss any models in the entire content. One extracted model JSON format should look like this: 
+                {"model_name": "GPT-4", "input_fee": "US$10.00 / 1M tokens", "output_fee": "US$30.00 / 1M tokens"}.""",
+                extra_args=extra_args,
+            ),
+            cache_mode=CacheMode.BYPASS,
+        )
+        print(result.extracted_content)
+
+
+async def extract_structured_data_using_css_extractor():
+    print("\n--- Using JsonCssExtractionStrategy for Fast Structured Output ---")
+    schema = {
+        "name": "KidoCode Courses",
+        "baseSelector": "section.charge-methodology .w-tab-content > div",
+        "fields": [
+            {
+                "name": "section_title",
+                "selector": "h3.heading-50",
+                "type": "text",
+            },
+            {
+                "name": "section_description",
+                "selector": ".charge-content",
+                "type": "text",
+            },
+            {
+                "name": "course_name",
+                "selector": ".text-block-93",
+                "type": "text",
+            },
+            {
+                "name": "course_description",
+                "selector": ".course-content-text",
+                "type": "text",
+            },
+            {
+                "name": "course_icon",
+                "selector": ".image-92",
+                "type": "attribute",
+                "attribute": "src",
+            },
+        ],
+    }
+
+    async with AsyncWebCrawler(headless=True, verbose=True) as crawler:
+        # Create the JavaScript that handles clicking multiple times
+        js_click_tabs = """
+        (async () => {
+            const tabs = document.querySelectorAll("section.charge-methodology .tabs-menu-3 > div");
+            
+            for(let tab of tabs) {
+                // scroll to the tab
+                tab.scrollIntoView();
+                tab.click();
+                // Wait for content to load and animations to complete
+                await new Promise(r => setTimeout(r, 500));
+            }
+        })();
+        """
+
+        result = await crawler.arun(
+            url="https://www.kidocode.com/degrees/technology",
+            extraction_strategy=JsonCssExtractionStrategy(schema, verbose=True),
+            js_code=[js_click_tabs],
+            cache_mode=CacheMode.BYPASS,
+        )
+
+        companies = json.loads(result.extracted_content)
+        print(f"Successfully extracted {len(companies)} companies")
+        print(json.dumps(companies[0], indent=2))
+
+
+# Advanced Session-Based Crawling with Dynamic Content 🔄
+async def crawl_dynamic_content_pages_method_1():
+    print("\n--- Advanced Multi-Page Crawling with JavaScript Execution ---")
+    first_commit = ""
+
+    async def on_execution_started(page):
+        nonlocal first_commit
+        try:
+            while True:
+                await page.wait_for_selector("li.Box-sc-g0xbh4-0 h4")
+                commit = await page.query_selector("li.Box-sc-g0xbh4-0 h4")
+                commit = await commit.evaluate("(element) => element.textContent")
+                commit = re.sub(r"\s+", "", commit)
+                if commit and commit != first_commit:
+                    first_commit = commit
+                    break
+                await asyncio.sleep(0.5)
+        except Exception as e:
+            print(f"Warning: New content didn't appear after JavaScript execution: {e}")
+
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        crawler.crawler_strategy.set_hook("on_execution_started", on_execution_started)
+
+        url = "https://github.com/microsoft/TypeScript/commits/main"
+        session_id = "typescript_commits_session"
+        all_commits = []
+
+        js_next_page = """
+        (() => {
+            const button = document.querySelector('a[data-testid="pagination-next-button"]');
+            if (button) button.click();
+        })();
+        """
+
+        for page in range(3):  # Crawl 3 pages
+            result = await crawler.arun(
+                url=url,
+                session_id=session_id,
+                css_selector="li.Box-sc-g0xbh4-0",
+                js=js_next_page if page > 0 else None,
+                cache_mode=CacheMode.BYPASS,
+                js_only=page > 0,
+                headless=False,
+            )
+
+            assert result.success, f"Failed to crawl page {page + 1}"
+
+            soup = BeautifulSoup(result.cleaned_html, "html.parser")
+            commits = soup.select("li")
+            all_commits.extend(commits)
+
+            print(f"Page {page + 1}: Found {len(commits)} commits")
+
+        await crawler.crawler_strategy.kill_session(session_id)
+        print(f"Successfully crawled {len(all_commits)} commits across 3 pages")
+
+
+async def crawl_dynamic_content_pages_method_2():
+    print("\n--- Advanced Multi-Page Crawling with JavaScript Execution ---")
+
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://github.com/microsoft/TypeScript/commits/main"
+        session_id = "typescript_commits_session"
+        all_commits = []
+        last_commit = ""
+
+        js_next_page_and_wait = """
+        (async () => {
+            const getCurrentCommit = () => {
+                const commits = document.querySelectorAll('li.Box-sc-g0xbh4-0 h4');
+                return commits.length > 0 ? commits[0].textContent.trim() : null;
+            };
+
+            const initialCommit = getCurrentCommit();
+            const button = document.querySelector('a[data-testid="pagination-next-button"]');
+            if (button) button.click();
+
+            // Poll for changes
+            while (true) {
+                await new Promise(resolve => setTimeout(resolve, 100)); // Wait 100ms
+                const newCommit = getCurrentCommit();
+                if (newCommit && newCommit !== initialCommit) {
+                    break;
+                }
+            }
+        })();
+        """
+
+        schema = {
+            "name": "Commit Extractor",
+            "baseSelector": "li.Box-sc-g0xbh4-0",
+            "fields": [
+                {
+                    "name": "title",
+                    "selector": "h4.markdown-title",
+                    "type": "text",
+                    "transform": "strip",
+                },
+            ],
+        }
+        extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True)
+
+        for page in range(3):  # Crawl 3 pages
+            result = await crawler.arun(
+                url=url,
+                session_id=session_id,
+                css_selector="li.Box-sc-g0xbh4-0",
+                extraction_strategy=extraction_strategy,
+                js_code=js_next_page_and_wait if page > 0 else None,
+                js_only=page > 0,
+                cache_mode=CacheMode.BYPASS,
+                headless=False,
+            )
+
+            assert result.success, f"Failed to crawl page {page + 1}"
+
+            commits = json.loads(result.extracted_content)
+            all_commits.extend(commits)
+
+            print(f"Page {page + 1}: Found {len(commits)} commits")
+
+        await crawler.crawler_strategy.kill_session(session_id)
+        print(f"Successfully crawled {len(all_commits)} commits across 3 pages")
+
+
+async def crawl_dynamic_content_pages_method_3():
+    print(
+        "\n--- Advanced Multi-Page Crawling with JavaScript Execution using `wait_for` ---"
+    )
+
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://github.com/microsoft/TypeScript/commits/main"
+        session_id = "typescript_commits_session"
+        all_commits = []
+
+        js_next_page = """
+        const commits = document.querySelectorAll('li.Box-sc-g0xbh4-0 h4');
+        if (commits.length > 0) {
+            window.firstCommit = commits[0].textContent.trim();
+        }
+        const button = document.querySelector('a[data-testid="pagination-next-button"]');
+        if (button) button.click();
+        """
+
+        wait_for = """() => {
+            const commits = document.querySelectorAll('li.Box-sc-g0xbh4-0 h4');
+            if (commits.length === 0) return false;
+            const firstCommit = commits[0].textContent.trim();
+            return firstCommit !== window.firstCommit;
+        }"""
+
+        schema = {
+            "name": "Commit Extractor",
+            "baseSelector": "li.Box-sc-g0xbh4-0",
+            "fields": [
+                {
+                    "name": "title",
+                    "selector": "h4.markdown-title",
+                    "type": "text",
+                    "transform": "strip",
+                },
+            ],
+        }
+        extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True)
+
+        for page in range(3):  # Crawl 3 pages
+            result = await crawler.arun(
+                url=url,
+                session_id=session_id,
+                css_selector="li.Box-sc-g0xbh4-0",
+                extraction_strategy=extraction_strategy,
+                js_code=js_next_page if page > 0 else None,
+                wait_for=wait_for if page > 0 else None,
+                js_only=page > 0,
+                cache_mode=CacheMode.BYPASS,
+                headless=False,
+            )
+
+            assert result.success, f"Failed to crawl page {page + 1}"
+
+            commits = json.loads(result.extracted_content)
+            all_commits.extend(commits)
+
+            print(f"Page {page + 1}: Found {len(commits)} commits")
+
+        await crawler.crawler_strategy.kill_session(session_id)
+        print(f"Successfully crawled {len(all_commits)} commits across 3 pages")
+
+
+async def crawl_custom_browser_type():
+    # Use Firefox
+    start = time.time()
+    async with AsyncWebCrawler(
+        browser_type="firefox", verbose=True, headless=True
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.example.com", cache_mode=CacheMode.BYPASS
+        )
+        print(result.markdown[:500])
+        print("Time taken: ", time.time() - start)
+
+    # Use WebKit
+    start = time.time()
+    async with AsyncWebCrawler(
+        browser_type="webkit", verbose=True, headless=True
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.example.com", cache_mode=CacheMode.BYPASS
+        )
+        print(result.markdown[:500])
+        print("Time taken: ", time.time() - start)
+
+    # Use Chromium (default)
+    start = time.time()
+    async with AsyncWebCrawler(verbose=True, headless=True) as crawler:
+        result = await crawler.arun(
+            url="https://www.example.com", cache_mode=CacheMode.BYPASS
+        )
+        print(result.markdown[:500])
+        print("Time taken: ", time.time() - start)
+
+
+async def crawl_with_user_simultion():
+    async with AsyncWebCrawler(verbose=True, headless=True) as crawler:
+        url = "YOUR-URL-HERE"
+        result = await crawler.arun(
+            url=url,
+            cache_mode=CacheMode.BYPASS,
+            magic=True,  # Automatically detects and removes overlays, popups, and other elements that block content
+            # simulate_user = True,# Causes a series of random mouse movements and clicks to simulate user interaction
+            # override_navigator = True # Overrides the navigator object to make it look like a real user
+        )
+
+        print(result.markdown)
+
+
+async def speed_comparison():
+    # print("\n--- Speed Comparison ---")
+    # print("Firecrawl (simulated):")
+    # print("Time taken: 7.02 seconds")
+    # print("Content length: 42074 characters")
+    # print("Images found: 49")
+    # print()
+    # Simulated Firecrawl performance
+    from firecrawl import FirecrawlApp
+
+    app = FirecrawlApp(api_key=os.environ["FIRECRAWL_API_KEY"])
+    start = time.time()
+    scrape_status = app.scrape_url(
+        "https://www.nbcnews.com/business", params={"formats": ["markdown", "html"]}
+    )
+    end = time.time()
+    print("Firecrawl:")
+    print(f"Time taken: {end - start:.2f} seconds")
+    print(f"Content length: {len(scrape_status['markdown'])} characters")
+    print(f"Images found: {scrape_status['markdown'].count('cldnry.s-nbcnews.com')}")
+    print()
+
+    async with AsyncWebCrawler() as crawler:
+        # Crawl4AI simple crawl
+        start = time.time()
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+            word_count_threshold=0,
+            cache_mode=CacheMode.BYPASS,
+            verbose=False,
+        )
+        end = time.time()
+        print("Crawl4AI (simple crawl):")
+        print(f"Time taken: {end - start:.2f} seconds")
+        print(f"Content length: {len(result.markdown)} characters")
+        print(f"Images found: {result.markdown.count('cldnry.s-nbcnews.com')}")
+        print()
+
+        # Crawl4AI with advanced content filtering
+        start = time.time()
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+            word_count_threshold=0,
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=PruningContentFilter(
+                    threshold=0.48, threshold_type="fixed", min_word_threshold=0
+                )
+                # content_filter=BM25ContentFilter(user_query=None, bm25_threshold=1.0)
+            ),
+            cache_mode=CacheMode.BYPASS,
+            verbose=False,
+        )
+        end = time.time()
+        print("Crawl4AI (Markdown Plus):")
+        print(f"Time taken: {end - start:.2f} seconds")
+        print(f"Content length: {len(result.markdown.raw_markdown)} characters")
+        print(f"Fit Markdown: {len(result.markdown.fit_markdown)} characters")
+        print(f"Images found: {result.markdown.raw_markdown.count('cldnry.s-nbcnews.com')}")
+        print()
+
+        # Crawl4AI with JavaScript execution
+        start = time.time()
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+            js_code=[
+                "const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"
+            ],
+            word_count_threshold=0,
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=PruningContentFilter(
+                    threshold=0.48, threshold_type="fixed", min_word_threshold=0
+                )
+                # content_filter=BM25ContentFilter(user_query=None, bm25_threshold=1.0)
+            ),
+            verbose=False,
+        )
+        end = time.time()
+        print("Crawl4AI (with JavaScript execution):")
+        print(f"Time taken: {end - start:.2f} seconds")
+        print(f"Content length: {len(result.markdown.raw_markdown)} characters")
+        print(f"Fit Markdown: {len(result.markdown.fit_markdown)} characters")
+        print(f"Images found: {result.markdown.raw_markdown.count('cldnry.s-nbcnews.com')}")
+
+    print("\nNote on Speed Comparison:")
+    print("The speed test conducted here may not reflect optimal conditions.")
+    print("When we call Firecrawl's API, we're seeing its best performance,")
+    print("while Crawl4AI's performance is limited by the local network speed.")
+    print("For a more accurate comparison, it's recommended to run these tests")
+    print("on servers with a stable and fast internet connection.")
+    print("Despite these limitations, Crawl4AI still demonstrates faster performance.")
+    print("If you run these tests in an environment with better network conditions,")
+    print("you may observe an even more significant speed advantage for Crawl4AI.")
+
+
+async def generate_knowledge_graph():
+    class Entity(BaseModel):
+        name: str
+        description: str
+
+    class Relationship(BaseModel):
+        entity1: Entity
+        entity2: Entity
+        description: str
+        relation_type: str
+
+    class KnowledgeGraph(BaseModel):
+        entities: List[Entity]
+        relationships: List[Relationship]
+
+    extraction_strategy = LLMExtractionStrategy(
+        llm_config=LLMConfig(provider="openai/gpt-4o-mini",  api_token=os.getenv("OPENAI_API_KEY")),  # In case of Ollama just pass "no-token"
+        schema=KnowledgeGraph.model_json_schema(),
+        extraction_type="schema",
+        instruction="""Extract entities and relationships from the given text.""",
+    )
+    async with AsyncWebCrawler() as crawler:
+        url = "https://paulgraham.com/love.html"
+        result = await crawler.arun(
+            url=url,
+            cache_mode=CacheMode.BYPASS,
+            extraction_strategy=extraction_strategy,
+            # magic=True
+        )
+        # print(result.extracted_content)
+        with open(os.path.join(__location__, "kb.json"), "w") as f:
+            f.write(result.extracted_content)
+
+
+async def fit_markdown_remove_overlay():
+    async with AsyncWebCrawler(
+        headless=True,  # Set to False to see what is happening
+        verbose=True,
+        user_agent_mode="random",
+        user_agent_generator_config={"device_type": "mobile", "os_type": "android"},
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.kidocode.com/degrees/technology",
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=PruningContentFilter(
+                    threshold=0.48, threshold_type="fixed", min_word_threshold=0
+                ),
+                options={"ignore_links": True},
+            ),
+            # markdown_generator=DefaultMarkdownGenerator(
+            #     content_filter=BM25ContentFilter(user_query="", bm25_threshold=1.0),
+            #     options={
+            #         "ignore_links": True
+            #     }
+            # ),
+        )
+
+        if result.success:
+            print(len(result.markdown.raw_markdown))
+            print(len(result.markdown.markdown_with_citations))
+            print(len(result.markdown.fit_markdown))
+
+            # Save clean html
+            with open(os.path.join(__location__, "output/cleaned_html.html"), "w") as f:
+                f.write(result.cleaned_html)
+
+            with open(
+                os.path.join(__location__, "output/output_raw_markdown.md"), "w"
+            ) as f:
+                f.write(result.markdown.raw_markdown)
+
+            with open(
+                os.path.join(__location__, "output/output_markdown_with_citations.md"),
+                "w",
+            ) as f:
+                f.write(result.markdown.markdown_with_citations)
+
+            with open(
+                os.path.join(__location__, "output/output_fit_markdown.md"), "w"
+            ) as f:
+                f.write(result.markdown.fit_markdown)
+
+    print("Done")
+
+
+async def main():
+    # await extract_structured_data_using_llm("openai/gpt-4o", os.getenv("OPENAI_API_KEY"))
+
+    # await simple_crawl()
+    # await simple_example_with_running_js_code()
+    # await simple_example_with_css_selector()
+    # # await use_proxy()
+    # await capture_and_save_screenshot("https://www.example.com", os.path.join(__location__, "tmp/example_screenshot.jpg"))
+    # await extract_structured_data_using_css_extractor()
+
+    # LLM extraction examples
+    # await extract_structured_data_using_llm()
+    # await extract_structured_data_using_llm("huggingface/meta-llama/Meta-Llama-3.1-8B-Instruct", os.getenv("HUGGINGFACE_API_KEY"))
+    # await extract_structured_data_using_llm("ollama/llama3.2")
+
+    # You always can pass custom headers to the extraction strategy
+    # custom_headers = {
+    #     "Authorization": "Bearer your-custom-token",
+    #     "X-Custom-Header": "Some-Value"
+    # }
+    # await extract_structured_data_using_llm(extra_headers=custom_headers)
+
+    # await crawl_dynamic_content_pages_method_1()
+    # await crawl_dynamic_content_pages_method_2()
+    await crawl_dynamic_content_pages_method_3()
+
+    # await crawl_custom_browser_type()
+
+    # await speed_comparison()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/quickstart_sync.py

@@ -0,0 +1,405 @@
+import os
+import time
+from crawl4ai import LLMConfig
+from crawl4ai.web_crawler import WebCrawler
+from crawl4ai.chunking_strategy import *
+from crawl4ai.extraction_strategy import *
+from crawl4ai.crawler_strategy import *
+from rich import print
+from rich.console import Console
+from functools import lru_cache
+
+console = Console()
+
+
+@lru_cache()
+def create_crawler():
+    crawler = WebCrawler(verbose=True)
+    crawler.warmup()
+    return crawler
+
+
+def print_result(result):
+    # Print each key in one line and just the first 10 characters of each one's value and three dots
+    console.print("\t[bold]Result:[/bold]")
+    for key, value in result.model_dump().items():
+        if isinstance(value, str) and value:
+            console.print(f"\t{key}: [green]{value[:20]}...[/green]")
+    if result.extracted_content:
+        items = json.loads(result.extracted_content)
+        print(f"\t[bold]{len(items)} blocks is extracted![/bold]")
+
+
+def cprint(message, press_any_key=False):
+    console.print(message)
+    if press_any_key:
+        console.print("Press any key to continue...", style="")
+        input()
+
+
+def basic_usage(crawler):
+    cprint(
+        "🛠️ [bold cyan]Basic Usage: Simply provide a URL and let Crawl4ai do the magic![/bold cyan]"
+    )
+    result = crawler.run(url="https://www.nbcnews.com/business", only_text=True)
+    cprint("[LOG] 📦 [bold yellow]Basic crawl result:[/bold yellow]")
+    print_result(result)
+
+
+def basic_usage_some_params(crawler):
+    cprint(
+        "🛠️ [bold cyan]Basic Usage: Simply provide a URL and let Crawl4ai do the magic![/bold cyan]"
+    )
+    result = crawler.run(
+        url="https://www.nbcnews.com/business", word_count_threshold=1, only_text=True
+    )
+    cprint("[LOG] 📦 [bold yellow]Basic crawl result:[/bold yellow]")
+    print_result(result)
+
+
+def screenshot_usage(crawler):
+    cprint("\n📸 [bold cyan]Let's take a screenshot of the page![/bold cyan]")
+    result = crawler.run(url="https://www.nbcnews.com/business", screenshot=True)
+    cprint("[LOG] 📦 [bold yellow]Screenshot result:[/bold yellow]")
+    # Save the screenshot to a file
+    with open("screenshot.png", "wb") as f:
+        f.write(base64.b64decode(result.screenshot))
+    cprint("Screenshot saved to 'screenshot.png'!")
+    print_result(result)
+
+
+def understanding_parameters(crawler):
+    cprint(
+        "\n🧠 [bold cyan]Understanding 'bypass_cache' and 'include_raw_html' parameters:[/bold cyan]"
+    )
+    cprint(
+        "By default, Crawl4ai caches the results of your crawls. This means that subsequent crawls of the same URL will be much faster! Let's see this in action."
+    )
+
+    # First crawl (reads from cache)
+    cprint("1️⃣ First crawl (caches the result):", True)
+    start_time = time.time()
+    result = crawler.run(url="https://www.nbcnews.com/business")
+    end_time = time.time()
+    cprint(
+        f"[LOG] 📦 [bold yellow]First crawl took {end_time - start_time} seconds and result (from cache):[/bold yellow]"
+    )
+    print_result(result)
+
+    # Force to crawl again
+    cprint("2️⃣ Second crawl (Force to crawl again):", True)
+    start_time = time.time()
+    result = crawler.run(url="https://www.nbcnews.com/business", bypass_cache=True)
+    end_time = time.time()
+    cprint(
+        f"[LOG] 📦 [bold yellow]Second crawl took {end_time - start_time} seconds and result (forced to crawl):[/bold yellow]"
+    )
+    print_result(result)
+
+
+def add_chunking_strategy(crawler):
+    # Adding a chunking strategy: RegexChunking
+    cprint(
+        "\n🧩 [bold cyan]Let's add a chunking strategy: RegexChunking![/bold cyan]",
+        True,
+    )
+    cprint(
+        "RegexChunking is a simple chunking strategy that splits the text based on a given regex pattern. Let's see it in action!"
+    )
+    result = crawler.run(
+        url="https://www.nbcnews.com/business",
+        chunking_strategy=RegexChunking(patterns=["\n\n"]),
+    )
+    cprint("[LOG] 📦 [bold yellow]RegexChunking result:[/bold yellow]")
+    print_result(result)
+
+    # Adding another chunking strategy: NlpSentenceChunking
+    cprint(
+        "\n🔍 [bold cyan]Time to explore another chunking strategy: NlpSentenceChunking![/bold cyan]",
+        True,
+    )
+    cprint(
+        "NlpSentenceChunking uses NLP techniques to split the text into sentences. Let's see how it performs!"
+    )
+    result = crawler.run(
+        url="https://www.nbcnews.com/business", chunking_strategy=NlpSentenceChunking()
+    )
+    cprint("[LOG] 📦 [bold yellow]NlpSentenceChunking result:[/bold yellow]")
+    print_result(result)
+
+
+def add_extraction_strategy(crawler):
+    # Adding an extraction strategy: CosineStrategy
+    cprint(
+        "\n🧠 [bold cyan]Let's get smarter with an extraction strategy: CosineStrategy![/bold cyan]",
+        True,
+    )
+    cprint(
+        "CosineStrategy uses cosine similarity to extract semantically similar blocks of text. Let's see it in action!"
+    )
+    result = crawler.run(
+        url="https://www.nbcnews.com/business",
+        extraction_strategy=CosineStrategy(
+            word_count_threshold=10,
+            max_dist=0.2,
+            linkage_method="ward",
+            top_k=3,
+            sim_threshold=0.3,
+            verbose=True,
+        ),
+    )
+    cprint("[LOG] 📦 [bold yellow]CosineStrategy result:[/bold yellow]")
+    print_result(result)
+
+    # Using semantic_filter with CosineStrategy
+    cprint(
+        "You can pass other parameters like 'semantic_filter' to the CosineStrategy to extract semantically similar blocks of text. Let's see it in action!"
+    )
+    result = crawler.run(
+        url="https://www.nbcnews.com/business",
+        extraction_strategy=CosineStrategy(
+            semantic_filter="inflation rent prices",
+        ),
+    )
+    cprint(
+        "[LOG] 📦 [bold yellow]CosineStrategy result with semantic filter:[/bold yellow]"
+    )
+    print_result(result)
+
+
+def add_llm_extraction_strategy(crawler):
+    # Adding an LLM extraction strategy without instructions
+    cprint(
+        "\n🤖 [bold cyan]Time to bring in the big guns: LLMExtractionStrategy without instructions![/bold cyan]",
+        True,
+    )
+    cprint(
+        "LLMExtractionStrategy uses a large language model to extract relevant information from the web page. Let's see it in action!"
+    )
+    result = crawler.run(
+        url="https://www.nbcnews.com/business",
+        extraction_strategy=LLMExtractionStrategy(
+            llm_config =  LLMConfig(provider="openai/gpt-4o", api_token=os.getenv("OPENAI_API_KEY"))
+        ),
+    )
+    cprint(
+        "[LOG] 📦 [bold yellow]LLMExtractionStrategy (no instructions) result:[/bold yellow]"
+    )
+    print_result(result)
+
+    # Adding an LLM extraction strategy with instructions
+    cprint(
+        "\n📜 [bold cyan]Let's make it even more interesting: LLMExtractionStrategy with instructions![/bold cyan]",
+        True,
+    )
+    cprint(
+        "Let's say we are only interested in financial news. Let's see how LLMExtractionStrategy performs with instructions!"
+    )
+    result = crawler.run(
+        url="https://www.nbcnews.com/business",
+        extraction_strategy=LLMExtractionStrategy(
+            llm_config=LLMConfig(provider="openai/gpt-4o",api_token=os.getenv("OPENAI_API_KEY")),
+            instruction="I am interested in only financial news",
+        ),
+    )
+    cprint(
+        "[LOG] 📦 [bold yellow]LLMExtractionStrategy (with instructions) result:[/bold yellow]"
+    )
+    print_result(result)
+
+    result = crawler.run(
+        url="https://www.nbcnews.com/business",
+        extraction_strategy=LLMExtractionStrategy(
+            llm_config=LLMConfig(provider="openai/gpt-4o",api_token=os.getenv("OPENAI_API_KEY")),
+            instruction="Extract only content related to technology",
+        ),
+    )
+    cprint(
+        "[LOG] 📦 [bold yellow]LLMExtractionStrategy (with technology instruction) result:[/bold yellow]"
+    )
+    print_result(result)
+
+
+def targeted_extraction(crawler):
+    # Using a CSS selector to extract only H2 tags
+    cprint(
+        "\n🎯 [bold cyan]Targeted extraction: Let's use a CSS selector to extract only H2 tags![/bold cyan]",
+        True,
+    )
+    result = crawler.run(url="https://www.nbcnews.com/business", css_selector="h2")
+    cprint("[LOG] 📦 [bold yellow]CSS Selector (H2 tags) result:[/bold yellow]")
+    print_result(result)
+
+
+def interactive_extraction(crawler):
+    # Passing JavaScript code to interact with the page
+    cprint(
+        "\n🖱️ [bold cyan]Let's get interactive: Passing JavaScript code to click 'Load More' button![/bold cyan]",
+        True,
+    )
+    cprint(
+        "In this example we try to click the 'Load More' button on the page using JavaScript code."
+    )
+    js_code = """
+    const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More'));
+    loadMoreButton && loadMoreButton.click();
+    """
+    # crawler_strategy = LocalSeleniumCrawlerStrategy(js_code=js_code)
+    # crawler = WebCrawler(crawler_strategy=crawler_strategy, always_by_pass_cache=True)
+    result = crawler.run(url="https://www.nbcnews.com/business", js=js_code)
+    cprint(
+        "[LOG] 📦 [bold yellow]JavaScript Code (Load More button) result:[/bold yellow]"
+    )
+    print_result(result)
+
+
+def multiple_scrip(crawler):
+    # Passing JavaScript code to interact with the page
+    cprint(
+        "\n🖱️ [bold cyan]Let's get interactive: Passing JavaScript code to click 'Load More' button![/bold cyan]",
+        True,
+    )
+    cprint(
+        "In this example we try to click the 'Load More' button on the page using JavaScript code."
+    )
+    js_code = [
+        """
+    const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More'));
+    loadMoreButton && loadMoreButton.click();
+    """
+    ] * 2
+    # crawler_strategy = LocalSeleniumCrawlerStrategy(js_code=js_code)
+    # crawler = WebCrawler(crawler_strategy=crawler_strategy, always_by_pass_cache=True)
+    result = crawler.run(url="https://www.nbcnews.com/business", js=js_code)
+    cprint(
+        "[LOG] 📦 [bold yellow]JavaScript Code (Load More button) result:[/bold yellow]"
+    )
+    print_result(result)
+
+
+def using_crawler_hooks(crawler):
+    # Example usage of the hooks for authentication and setting a cookie
+    def on_driver_created(driver):
+        print("[HOOK] on_driver_created")
+        # Example customization: maximize the window
+        driver.maximize_window()
+
+        # Example customization: logging in to a hypothetical website
+        driver.get("https://example.com/login")
+
+        from selenium.webdriver.support.ui import WebDriverWait
+        from selenium.webdriver.common.by import By
+        from selenium.webdriver.support import expected_conditions as EC
+
+        WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located((By.NAME, "username"))
+        )
+        driver.find_element(By.NAME, "username").send_keys("testuser")
+        driver.find_element(By.NAME, "password").send_keys("password123")
+        driver.find_element(By.NAME, "login").click()
+        WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located((By.ID, "welcome"))
+        )
+        # Add a custom cookie
+        driver.add_cookie({"name": "test_cookie", "value": "cookie_value"})
+        return driver
+
+    def before_get_url(driver):
+        print("[HOOK] before_get_url")
+        # Example customization: add a custom header
+        # Enable Network domain for sending headers
+        driver.execute_cdp_cmd("Network.enable", {})
+        # Add a custom header
+        driver.execute_cdp_cmd(
+            "Network.setExtraHTTPHeaders", {"headers": {"X-Test-Header": "test"}}
+        )
+        return driver
+
+    def after_get_url(driver):
+        print("[HOOK] after_get_url")
+        # Example customization: log the URL
+        print(driver.current_url)
+        return driver
+
+    def before_return_html(driver, html):
+        print("[HOOK] before_return_html")
+        # Example customization: log the HTML
+        print(len(html))
+        return driver
+
+    cprint(
+        "\n🔗 [bold cyan]Using Crawler Hooks: Let's see how we can customize the crawler using hooks![/bold cyan]",
+        True,
+    )
+
+    crawler_strategy = LocalSeleniumCrawlerStrategy(verbose=True)
+    crawler_strategy.set_hook("on_driver_created", on_driver_created)
+    crawler_strategy.set_hook("before_get_url", before_get_url)
+    crawler_strategy.set_hook("after_get_url", after_get_url)
+    crawler_strategy.set_hook("before_return_html", before_return_html)
+
+    crawler = WebCrawler(verbose=True, crawler_strategy=crawler_strategy)
+    crawler.warmup()
+    result = crawler.run(url="https://example.com")
+
+    cprint("[LOG] 📦 [bold yellow]Crawler Hooks result:[/bold yellow]")
+    print_result(result=result)
+
+
+def using_crawler_hooks_dleay_example(crawler):
+    def delay(driver):
+        print("Delaying for 5 seconds...")
+        time.sleep(5)
+        print("Resuming...")
+
+    def create_crawler():
+        crawler_strategy = LocalSeleniumCrawlerStrategy(verbose=True)
+        crawler_strategy.set_hook("after_get_url", delay)
+        crawler = WebCrawler(verbose=True, crawler_strategy=crawler_strategy)
+        crawler.warmup()
+        return crawler
+
+    cprint(
+        "\n🔗 [bold cyan]Using Crawler Hooks: Let's add a delay after fetching the url to make sure entire page is fetched.[/bold cyan]"
+    )
+    crawler = create_crawler()
+    result = crawler.run(url="https://google.com", bypass_cache=True)
+
+    cprint("[LOG] 📦 [bold yellow]Crawler Hooks result:[/bold yellow]")
+    print_result(result)
+
+
+def main():
+    cprint(
+        "🌟 [bold green]Welcome to the Crawl4ai Quickstart Guide! Let's dive into some web crawling fun! 🌐[/bold green]"
+    )
+    cprint(
+        "⛳️ [bold cyan]First Step: Create an instance of WebCrawler and call the `warmup()` function.[/bold cyan]"
+    )
+    cprint(
+        "If this is the first time you're running Crawl4ai, this might take a few seconds to load required model files."
+    )
+
+    crawler = create_crawler()
+
+    crawler.always_by_pass_cache = True
+    basic_usage(crawler)
+    # basic_usage_some_params(crawler)
+    understanding_parameters(crawler)
+
+    crawler.always_by_pass_cache = True
+    screenshot_usage(crawler)
+    add_chunking_strategy(crawler)
+    add_extraction_strategy(crawler)
+    add_llm_extraction_strategy(crawler)
+    targeted_extraction(crawler)
+    interactive_extraction(crawler)
+    multiple_scrip(crawler)
+
+    cprint(
+        "\n🎉 [bold green]Congratulations! You've made it through the Crawl4ai Quickstart Guide! Now go forth and crawl the web like a pro! 🕸️[/bold green]"
+    )
+
+
+if __name__ == "__main__":
+    main()

docs/examples/quickstart_v0.ipynb

@@ -0,0 +1,735 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6yLvrXn7yZQI"
+      },
+      "source": [
+        "# Crawl4AI: Advanced Web Crawling and Data Extraction\n",
+        "\n",
+        "Welcome to this interactive notebook showcasing Crawl4AI, an advanced asynchronous web crawling and data extraction library.\n",
+        "\n",
+        "- GitHub Repository: [https://github.com/unclecode/crawl4ai](https://github.com/unclecode/crawl4ai)\n",
+        "- Twitter: [@unclecode](https://twitter.com/unclecode)\n",
+        "- Website: [https://crawl4ai.com](https://crawl4ai.com)\n",
+        "\n",
+        "Let's explore the powerful features of Crawl4AI!"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KIn_9nxFyZQK"
+      },
+      "source": [
+        "## Installation\n",
+        "\n",
+        "First, let's install Crawl4AI from GitHub:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mSnaxLf3zMog"
+      },
+      "outputs": [],
+      "source": [
+        "!sudo apt-get update && sudo apt-get install -y libwoff1 libopus0 libwebp6 libwebpdemux2 libenchant1c2a libgudev-1.0-0 libsecret-1-0 libhyphen0 libgdk-pixbuf2.0-0 libegl1 libnotify4 libxslt1.1 libevent-2.1-7 libgles2 libvpx6 libxcomposite1 libatk1.0-0 libatk-bridge2.0-0 libepoxy0 libgtk-3-0 libharfbuzz-icu0"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xlXqaRtayZQK"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install crawl4ai\n",
+        "!pip install nest-asyncio\n",
+        "!playwright install"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qKCE7TI7yZQL"
+      },
+      "source": [
+        "Now, let's import the necessary libraries:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {
+        "id": "I67tr7aAyZQL"
+      },
+      "outputs": [],
+      "source": [
+        "import asyncio\n",
+        "import nest_asyncio\n",
+        "from crawl4ai import AsyncWebCrawler\n",
+        "from crawl4ai.extraction_strategy import JsonCssExtractionStrategy, LLMExtractionStrategy\n",
+        "import json\n",
+        "import time\n",
+        "from pydantic import BaseModel, Field\n",
+        "\n",
+        "nest_asyncio.apply()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "h7yR_Rt_yZQM"
+      },
+      "source": [
+        "## Basic Usage\n",
+        "\n",
+        "Let's start with a simple crawl example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "yBh6hf4WyZQM",
+        "outputId": "0f83af5c-abba-4175-ed95-70b7512e6bcc"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "[LOG] 🌤️  Warming up the AsyncWebCrawler\n",
+            "[LOG] 🌞 AsyncWebCrawler is ready to crawl\n",
+            "[LOG] 🚀 Content extracted for https://www.nbcnews.com/business, success: True, time taken: 0.05 seconds\n",
+            "[LOG] 🚀 Extraction done for https://www.nbcnews.com/business, time taken: 0.05 seconds.\n",
+            "18102\n"
+          ]
+        }
+      ],
+      "source": [
+        "async def simple_crawl():\n",
+        "    async with AsyncWebCrawler(verbose=True) as crawler:\n",
+        "        result = await crawler.arun(url=\"https://www.nbcnews.com/business\")\n",
+        "        print(len(result.markdown))\n",
+        "await simple_crawl()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9rtkgHI28uI4"
+      },
+      "source": [
+        "💡 By default, **Crawl4AI** caches the result of every URL, so the next time you call it, you’ll get an instant result. But if you want to bypass the cache, just set `bypass_cache=True`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MzZ0zlJ9yZQM"
+      },
+      "source": [
+        "## Advanced Features\n",
+        "\n",
+        "### Executing JavaScript and Using CSS Selectors"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 3,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "gHStF86xyZQM",
+        "outputId": "34d0fb6d-4dec-4677-f76e-85a1f082829b"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "[LOG] 🌤️  Warming up the AsyncWebCrawler\n",
+            "[LOG] 🌞 AsyncWebCrawler is ready to crawl\n",
+            "[LOG] 🕸️ Crawling https://www.nbcnews.com/business using AsyncPlaywrightCrawlerStrategy...\n",
+            "[LOG] ✅ Crawled https://www.nbcnews.com/business successfully!\n",
+            "[LOG] 🚀 Crawling done for https://www.nbcnews.com/business, success: True, time taken: 6.06 seconds\n",
+            "[LOG] 🚀 Content extracted for https://www.nbcnews.com/business, success: True, time taken: 0.10 seconds\n",
+            "[LOG] 🔥 Extracting semantic blocks for https://www.nbcnews.com/business, Strategy: AsyncWebCrawler\n",
+            "[LOG] 🚀 Extraction done for https://www.nbcnews.com/business, time taken: 0.11 seconds.\n",
+            "41135\n"
+          ]
+        }
+      ],
+      "source": [
+        "async def js_and_css():\n",
+        "    async with AsyncWebCrawler(verbose=True) as crawler:\n",
+        "        js_code = [\"const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();\"]\n",
+        "        result = await crawler.arun(\n",
+        "            url=\"https://www.nbcnews.com/business\",\n",
+        "            js_code=js_code,\n",
+        "            # css_selector=\"YOUR_CSS_SELECTOR_HERE\",\n",
+        "            bypass_cache=True\n",
+        "        )\n",
+        "        print(len(result.markdown))\n",
+        "\n",
+        "await js_and_css()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cqE_W4coyZQM"
+      },
+      "source": [
+        "### Using a Proxy\n",
+        "\n",
+        "Note: You'll need to replace the proxy URL with a working proxy for this example to run successfully."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QjAyiAGqyZQM"
+      },
+      "outputs": [],
+      "source": [
+        "async def use_proxy():\n",
+        "    async with AsyncWebCrawler(verbose=True, proxy=\"http://your-proxy-url:port\") as crawler:\n",
+        "        result = await crawler.arun(\n",
+        "            url=\"https://www.nbcnews.com/business\",\n",
+        "            bypass_cache=True\n",
+        "        )\n",
+        "        print(result.markdown[:500])  # Print first 500 characters\n",
+        "\n",
+        "# Uncomment the following line to run the proxy example\n",
+        "# await use_proxy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XTZ88lbayZQN"
+      },
+      "source": [
+        "### Extracting Structured Data with OpenAI\n",
+        "\n",
+        "Note: You'll need to set your OpenAI API key as an environment variable for this example to work."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 14,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "fIOlDayYyZQN",
+        "outputId": "cb8359cc-dee0-4762-9698-5dfdcee055b8"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "[LOG] 🌤️  Warming up the AsyncWebCrawler\n",
+            "[LOG] 🌞 AsyncWebCrawler is ready to crawl\n",
+            "[LOG] 🕸️ Crawling https://openai.com/api/pricing/ using AsyncPlaywrightCrawlerStrategy...\n",
+            "[LOG] ✅ Crawled https://openai.com/api/pricing/ successfully!\n",
+            "[LOG] 🚀 Crawling done for https://openai.com/api/pricing/, success: True, time taken: 3.77 seconds\n",
+            "[LOG] 🚀 Content extracted for https://openai.com/api/pricing/, success: True, time taken: 0.21 seconds\n",
+            "[LOG] 🔥 Extracting semantic blocks for https://openai.com/api/pricing/, Strategy: AsyncWebCrawler\n",
+            "[LOG] Call LLM for https://openai.com/api/pricing/ - block index: 0\n",
+            "[LOG] Call LLM for https://openai.com/api/pricing/ - block index: 1\n",
+            "[LOG] Call LLM for https://openai.com/api/pricing/ - block index: 2\n",
+            "[LOG] Call LLM for https://openai.com/api/pricing/ - block index: 3\n",
+            "[LOG] Extracted 4 blocks from URL: https://openai.com/api/pricing/ block index: 3\n",
+            "[LOG] Call LLM for https://openai.com/api/pricing/ - block index: 4\n",
+            "[LOG] Extracted 5 blocks from URL: https://openai.com/api/pricing/ block index: 0\n",
+            "[LOG] Extracted 1 blocks from URL: https://openai.com/api/pricing/ block index: 4\n",
+            "[LOG] Extracted 8 blocks from URL: https://openai.com/api/pricing/ block index: 1\n",
+            "[LOG] Extracted 12 blocks from URL: https://openai.com/api/pricing/ block index: 2\n",
+            "[LOG] 🚀 Extraction done for https://openai.com/api/pricing/, time taken: 8.55 seconds.\n",
+            "5029\n"
+          ]
+        }
+      ],
+      "source": [
+        "import os\n",
+        "from google.colab import userdata\n",
+        "os.environ['OPENAI_API_KEY'] = userdata.get('OPENAI_API_KEY')\n",
+        "\n",
+        "class OpenAIModelFee(BaseModel):\n",
+        "    model_name: str = Field(..., description=\"Name of the OpenAI model.\")\n",
+        "    input_fee: str = Field(..., description=\"Fee for input token for the OpenAI model.\")\n",
+        "    output_fee: str = Field(..., description=\"Fee for output token for the OpenAI model.\")\n",
+        "\n",
+        "async def extract_openai_fees():\n",
+        "    async with AsyncWebCrawler(verbose=True) as crawler:\n",
+        "        result = await crawler.arun(\n",
+        "            url='https://openai.com/api/pricing/',\n",
+        "            word_count_threshold=1,\n",
+        "            extraction_strategy=LLMExtractionStrategy(\n",
+        "                provider=\"openai/gpt-4o\", api_token=os.getenv('OPENAI_API_KEY'),\n",
+        "                schema=OpenAIModelFee.schema(),\n",
+        "                extraction_type=\"schema\",\n",
+        "                instruction=\"\"\"From the crawled content, extract all mentioned model names along with their fees for input and output tokens.\n",
+        "                Do not miss any models in the entire content. One extracted model JSON format should look like this:\n",
+        "                {\"model_name\": \"GPT-4\", \"input_fee\": \"US$10.00 / 1M tokens\", \"output_fee\": \"US$30.00 / 1M tokens\"}.\"\"\"\n",
+        "            ),\n",
+        "            bypass_cache=True,\n",
+        "        )\n",
+        "        print(len(result.extracted_content))\n",
+        "\n",
+        "# Uncomment the following line to run the OpenAI extraction example\n",
+        "await extract_openai_fees()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BypA5YxEyZQN"
+      },
+      "source": [
+        "### Advanced Multi-Page Crawling with JavaScript Execution"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tfkcVQ0b7mw-"
+      },
+      "source": [
+        "## Advanced Multi-Page Crawling with JavaScript Execution\n",
+        "\n",
+        "This example demonstrates Crawl4AI's ability to handle complex crawling scenarios, specifically extracting commits from multiple pages of a GitHub repository. The challenge here is that clicking the \"Next\" button doesn't load a new page, but instead uses asynchronous JavaScript to update the content. This is a common hurdle in modern web crawling.\n",
+        "\n",
+        "To overcome this, we use Crawl4AI's custom JavaScript execution to simulate clicking the \"Next\" button, and implement a custom hook to detect when new data has loaded. Our strategy involves comparing the first commit's text before and after \"clicking\" Next, waiting until it changes to confirm new data has rendered. This showcases Crawl4AI's flexibility in handling dynamic content and its ability to implement custom logic for even the most challenging crawling tasks."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 11,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "qUBKGpn3yZQN",
+        "outputId": "3e555b6a-ed33-42f4-cce9-499a923fbe17"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "[LOG] 🌤️  Warming up the AsyncWebCrawler\n",
+            "[LOG] 🌞 AsyncWebCrawler is ready to crawl\n",
+            "[LOG] 🕸️ Crawling https://github.com/microsoft/TypeScript/commits/main using AsyncPlaywrightCrawlerStrategy...\n",
+            "[LOG] ✅ Crawled https://github.com/microsoft/TypeScript/commits/main successfully!\n",
+            "[LOG] 🚀 Crawling done for https://github.com/microsoft/TypeScript/commits/main, success: True, time taken: 5.16 seconds\n",
+            "[LOG] 🚀 Content extracted for https://github.com/microsoft/TypeScript/commits/main, success: True, time taken: 0.28 seconds\n",
+            "[LOG] 🔥 Extracting semantic blocks for https://github.com/microsoft/TypeScript/commits/main, Strategy: AsyncWebCrawler\n",
+            "[LOG] 🚀 Extraction done for https://github.com/microsoft/TypeScript/commits/main, time taken: 0.28 seconds.\n",
+            "Page 1: Found 35 commits\n",
+            "[LOG] 🕸️ Crawling https://github.com/microsoft/TypeScript/commits/main using AsyncPlaywrightCrawlerStrategy...\n",
+            "[LOG] ✅ Crawled https://github.com/microsoft/TypeScript/commits/main successfully!\n",
+            "[LOG] 🚀 Crawling done for https://github.com/microsoft/TypeScript/commits/main, success: True, time taken: 0.78 seconds\n",
+            "[LOG] 🚀 Content extracted for https://github.com/microsoft/TypeScript/commits/main, success: True, time taken: 0.90 seconds\n",
+            "[LOG] 🔥 Extracting semantic blocks for https://github.com/microsoft/TypeScript/commits/main, Strategy: AsyncWebCrawler\n",
+            "[LOG] 🚀 Extraction done for https://github.com/microsoft/TypeScript/commits/main, time taken: 0.90 seconds.\n",
+            "Page 2: Found 35 commits\n",
+            "[LOG] 🕸️ Crawling https://github.com/microsoft/TypeScript/commits/main using AsyncPlaywrightCrawlerStrategy...\n",
+            "[LOG] ✅ Crawled https://github.com/microsoft/TypeScript/commits/main successfully!\n",
+            "[LOG] 🚀 Crawling done for https://github.com/microsoft/TypeScript/commits/main, success: True, time taken: 2.00 seconds\n",
+            "[LOG] 🚀 Content extracted for https://github.com/microsoft/TypeScript/commits/main, success: True, time taken: 0.74 seconds\n",
+            "[LOG] 🔥 Extracting semantic blocks for https://github.com/microsoft/TypeScript/commits/main, Strategy: AsyncWebCrawler\n",
+            "[LOG] 🚀 Extraction done for https://github.com/microsoft/TypeScript/commits/main, time taken: 0.75 seconds.\n",
+            "Page 3: Found 35 commits\n",
+            "Successfully crawled 105 commits across 3 pages\n"
+          ]
+        }
+      ],
+      "source": [
+        "import re\n",
+        "from bs4 import BeautifulSoup\n",
+        "\n",
+        "async def crawl_typescript_commits():\n",
+        "    first_commit = \"\"\n",
+        "    async def on_execution_started(page):\n",
+        "        nonlocal first_commit\n",
+        "        try:\n",
+        "            while True:\n",
+        "                await page.wait_for_selector('li.Box-sc-g0xbh4-0 h4')\n",
+        "                commit = await page.query_selector('li.Box-sc-g0xbh4-0 h4')\n",
+        "                commit = await commit.evaluate('(element) => element.textContent')\n",
+        "                commit = re.sub(r'\\s+', '', commit)\n",
+        "                if commit and commit != first_commit:\n",
+        "                    first_commit = commit\n",
+        "                    break\n",
+        "                await asyncio.sleep(0.5)\n",
+        "        except Exception as e:\n",
+        "            print(f\"Warning: New content didn't appear after JavaScript execution: {e}\")\n",
+        "\n",
+        "    async with AsyncWebCrawler(verbose=True) as crawler:\n",
+        "        crawler.crawler_strategy.set_hook('on_execution_started', on_execution_started)\n",
+        "\n",
+        "        url = \"https://github.com/microsoft/TypeScript/commits/main\"\n",
+        "        session_id = \"typescript_commits_session\"\n",
+        "        all_commits = []\n",
+        "\n",
+        "        js_next_page = \"\"\"\n",
+        "        const button = document.querySelector('a[data-testid=\"pagination-next-button\"]');\n",
+        "        if (button) button.click();\n",
+        "        \"\"\"\n",
+        "\n",
+        "        for page in range(3):  # Crawl 3 pages\n",
+        "            result = await crawler.arun(\n",
+        "                url=url,\n",
+        "                session_id=session_id,\n",
+        "                css_selector=\"li.Box-sc-g0xbh4-0\",\n",
+        "                js=js_next_page if page > 0 else None,\n",
+        "                bypass_cache=True,\n",
+        "                js_only=page > 0\n",
+        "            )\n",
+        "\n",
+        "            assert result.success, f\"Failed to crawl page {page + 1}\"\n",
+        "\n",
+        "            soup = BeautifulSoup(result.cleaned_html, 'html.parser')\n",
+        "            commits = soup.select(\"li\")\n",
+        "            all_commits.extend(commits)\n",
+        "\n",
+        "            print(f\"Page {page + 1}: Found {len(commits)} commits\")\n",
+        "\n",
+        "        await crawler.crawler_strategy.kill_session(session_id)\n",
+        "        print(f\"Successfully crawled {len(all_commits)} commits across 3 pages\")\n",
+        "\n",
+        "await crawl_typescript_commits()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EJRnYsp6yZQN"
+      },
+      "source": [
+        "### Using JsonCssExtractionStrategy for Fast Structured Output"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1ZMqIzB_8SYp"
+      },
+      "source": [
+        "The JsonCssExtractionStrategy is a powerful feature of Crawl4AI that allows for precise, structured data extraction from web pages. Here's how it works:\n",
+        "\n",
+        "1. You define a schema that describes the pattern of data you're interested in extracting.\n",
+        "2. The schema includes a base selector that identifies repeating elements on the page.\n",
+        "3. Within the schema, you define fields, each with its own selector and type.\n",
+        "4. These field selectors are applied within the context of each base selector element.\n",
+        "5. The strategy supports nested structures, lists within lists, and various data types.\n",
+        "6. You can even include computed fields for more complex data manipulation.\n",
+        "\n",
+        "This approach allows for highly flexible and precise data extraction, transforming semi-structured web content into clean, structured JSON data. It's particularly useful for extracting consistent data patterns from pages like product listings, news articles, or search results.\n",
+        "\n",
+        "For more details and advanced usage, check out the full documentation on the Crawl4AI website."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 12,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "trCMR2T9yZQN",
+        "outputId": "718d36f4-cccf-40f4-8d8c-c3ba73524d16"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "[LOG] 🌤️  Warming up the AsyncWebCrawler\n",
+            "[LOG] 🌞 AsyncWebCrawler is ready to crawl\n",
+            "[LOG] 🕸️ Crawling https://www.nbcnews.com/business using AsyncPlaywrightCrawlerStrategy...\n",
+            "[LOG] ✅ Crawled https://www.nbcnews.com/business successfully!\n",
+            "[LOG] 🚀 Crawling done for https://www.nbcnews.com/business, success: True, time taken: 7.00 seconds\n",
+            "[LOG] 🚀 Content extracted for https://www.nbcnews.com/business, success: True, time taken: 0.32 seconds\n",
+            "[LOG] 🔥 Extracting semantic blocks for https://www.nbcnews.com/business, Strategy: AsyncWebCrawler\n",
+            "[LOG] 🚀 Extraction done for https://www.nbcnews.com/business, time taken: 0.48 seconds.\n",
+            "Successfully extracted 11 news teasers\n",
+            "{\n",
+            "  \"category\": \"Business News\",\n",
+            "  \"headline\": \"NBC ripped up its Olympics playbook for 2024 \\u2014 so far, the new strategy paid off\",\n",
+            "  \"summary\": \"The Olympics have long been key to NBCUniversal. Paris marked the 18th Olympic Games broadcast by NBC in the U.S.\",\n",
+            "  \"time\": \"13h ago\",\n",
+            "  \"image\": {\n",
+            "    \"src\": \"https://media-cldnry.s-nbcnews.com/image/upload/t_focal-200x100,f_auto,q_auto:best/rockcms/2024-09/240903-nbc-olympics-ch-1344-c7a486.jpg\",\n",
+            "    \"alt\": \"Mike Tirico.\"\n",
+            "  },\n",
+            "  \"link\": \"https://www.nbcnews.com/business\"\n",
+            "}\n"
+          ]
+        }
+      ],
+      "source": [
+        "async def extract_news_teasers():\n",
+        "    schema = {\n",
+        "        \"name\": \"News Teaser Extractor\",\n",
+        "        \"baseSelector\": \".wide-tease-item__wrapper\",\n",
+        "        \"fields\": [\n",
+        "            {\n",
+        "                \"name\": \"category\",\n",
+        "                \"selector\": \".unibrow span[data-testid='unibrow-text']\",\n",
+        "                \"type\": \"text\",\n",
+        "            },\n",
+        "            {\n",
+        "                \"name\": \"headline\",\n",
+        "                \"selector\": \".wide-tease-item__headline\",\n",
+        "                \"type\": \"text\",\n",
+        "            },\n",
+        "            {\n",
+        "                \"name\": \"summary\",\n",
+        "                \"selector\": \".wide-tease-item__description\",\n",
+        "                \"type\": \"text\",\n",
+        "            },\n",
+        "            {\n",
+        "                \"name\": \"time\",\n",
+        "                \"selector\": \"[data-testid='wide-tease-date']\",\n",
+        "                \"type\": \"text\",\n",
+        "            },\n",
+        "            {\n",
+        "                \"name\": \"image\",\n",
+        "                \"type\": \"nested\",\n",
+        "                \"selector\": \"picture.teasePicture img\",\n",
+        "                \"fields\": [\n",
+        "                    {\"name\": \"src\", \"type\": \"attribute\", \"attribute\": \"src\"},\n",
+        "                    {\"name\": \"alt\", \"type\": \"attribute\", \"attribute\": \"alt\"},\n",
+        "                ],\n",
+        "            },\n",
+        "            {\n",
+        "                \"name\": \"link\",\n",
+        "                \"selector\": \"a[href]\",\n",
+        "                \"type\": \"attribute\",\n",
+        "                \"attribute\": \"href\",\n",
+        "            },\n",
+        "        ],\n",
+        "    }\n",
+        "\n",
+        "    extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True)\n",
+        "\n",
+        "    async with AsyncWebCrawler(verbose=True) as crawler:\n",
+        "        result = await crawler.arun(\n",
+        "            url=\"https://www.nbcnews.com/business\",\n",
+        "            extraction_strategy=extraction_strategy,\n",
+        "            bypass_cache=True,\n",
+        "        )\n",
+        "\n",
+        "        assert result.success, \"Failed to crawl the page\"\n",
+        "\n",
+        "        news_teasers = json.loads(result.extracted_content)\n",
+        "        print(f\"Successfully extracted {len(news_teasers)} news teasers\")\n",
+        "        print(json.dumps(news_teasers[0], indent=2))\n",
+        "\n",
+        "await extract_news_teasers()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FnyVhJaByZQN"
+      },
+      "source": [
+        "## Speed Comparison\n",
+        "\n",
+        "Let's compare the speed of Crawl4AI with Firecrawl, a paid service. Note that we can't run Firecrawl in this Colab environment, so we'll simulate its performance based on previously recorded data."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "agDD186f3wig"
+      },
+      "source": [
+        "💡 **Note on Speed Comparison:**\n",
+        "\n",
+        "The speed test conducted here is running on Google Colab, where the internet speed and performance can vary and may not reflect optimal conditions. When we call Firecrawl's API, we're seeing its best performance, while Crawl4AI's performance is limited by Colab's network speed.\n",
+        "\n",
+        "For a more accurate comparison, it's recommended to run these tests on your own servers or computers with a stable and fast internet connection. Despite these limitations, Crawl4AI still demonstrates faster performance in this environment.\n",
+        "\n",
+        "If you run these tests locally, you may observe an even more significant speed advantage for Crawl4AI compared to other services."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "F7KwHv8G1LbY"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install firecrawl"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 4,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "91813zILyZQN",
+        "outputId": "663223db-ab89-4976-b233-05ceca62b19b"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Firecrawl (simulated):\n",
+            "Time taken: 4.38 seconds\n",
+            "Content length: 41967 characters\n",
+            "Images found: 49\n",
+            "\n",
+            "Crawl4AI (simple crawl):\n",
+            "Time taken: 4.22 seconds\n",
+            "Content length: 18221 characters\n",
+            "Images found: 49\n",
+            "\n",
+            "Crawl4AI (with JavaScript execution):\n",
+            "Time taken: 9.13 seconds\n",
+            "Content length: 34243 characters\n",
+            "Images found: 89\n"
+          ]
+        }
+      ],
+      "source": [
+        "import os\n",
+        "from google.colab import userdata\n",
+        "os.environ['FIRECRAWL_API_KEY'] = userdata.get('FIRECRAWL_API_KEY')\n",
+        "import time\n",
+        "from firecrawl import FirecrawlApp\n",
+        "\n",
+        "async def speed_comparison():\n",
+        "    # Simulated Firecrawl performance\n",
+        "    app = FirecrawlApp(api_key=os.environ['FIRECRAWL_API_KEY'])\n",
+        "    start = time.time()\n",
+        "    scrape_status = app.scrape_url(\n",
+        "    'https://www.nbcnews.com/business',\n",
+        "    params={'formats': ['markdown', 'html']}\n",
+        "    )\n",
+        "    end = time.time()\n",
+        "    print(\"Firecrawl (simulated):\")\n",
+        "    print(f\"Time taken: {end - start:.2f} seconds\")\n",
+        "    print(f\"Content length: {len(scrape_status['markdown'])} characters\")\n",
+        "    print(f\"Images found: {scrape_status['markdown'].count('cldnry.s-nbcnews.com')}\")\n",
+        "    print()\n",
+        "\n",
+        "    async with AsyncWebCrawler() as crawler:\n",
+        "        # Crawl4AI simple crawl\n",
+        "        start = time.time()\n",
+        "        result = await crawler.arun(\n",
+        "            url=\"https://www.nbcnews.com/business\",\n",
+        "            word_count_threshold=0,\n",
+        "            bypass_cache=True,\n",
+        "            verbose=False\n",
+        "        )\n",
+        "        end = time.time()\n",
+        "        print(\"Crawl4AI (simple crawl):\")\n",
+        "        print(f\"Time taken: {end - start:.2f} seconds\")\n",
+        "        print(f\"Content length: {len(result.markdown)} characters\")\n",
+        "        print(f\"Images found: {result.markdown.count('cldnry.s-nbcnews.com')}\")\n",
+        "        print()\n",
+        "\n",
+        "        # Crawl4AI with JavaScript execution\n",
+        "        start = time.time()\n",
+        "        result = await crawler.arun(\n",
+        "            url=\"https://www.nbcnews.com/business\",\n",
+        "            js_code=[\"const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();\"],\n",
+        "            word_count_threshold=0,\n",
+        "            bypass_cache=True,\n",
+        "            verbose=False\n",
+        "        )\n",
+        "        end = time.time()\n",
+        "        print(\"Crawl4AI (with JavaScript execution):\")\n",
+        "        print(f\"Time taken: {end - start:.2f} seconds\")\n",
+        "        print(f\"Content length: {len(result.markdown)} characters\")\n",
+        "        print(f\"Images found: {result.markdown.count('cldnry.s-nbcnews.com')}\")\n",
+        "\n",
+        "await speed_comparison()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OBFFYVJIyZQN"
+      },
+      "source": [
+        "If you run on a local machine with a proper internet speed:\n",
+        "- Simple crawl: Crawl4AI is typically over 3-4 times faster than Firecrawl.\n",
+        "- With JavaScript execution: Even when executing JavaScript to load more content (potentially doubling the number of images found), Crawl4AI is still faster than Firecrawl's simple crawl.\n",
+        "\n",
+        "Please note that actual performance may vary depending on network conditions and the specific content being crawled."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A6_1RK1_yZQO"
+      },
+      "source": [
+        "## Conclusion\n",
+        "\n",
+        "In this notebook, we've explored the powerful features of Crawl4AI, including:\n",
+        "\n",
+        "1. Basic crawling\n",
+        "2. JavaScript execution and CSS selector usage\n",
+        "3. Proxy support\n",
+        "4. Structured data extraction with OpenAI\n",
+        "5. Advanced multi-page crawling with JavaScript execution\n",
+        "6. Fast structured output using JsonCssExtractionStrategy\n",
+        "7. Speed comparison with other services\n",
+        "\n",
+        "Crawl4AI offers a fast, flexible, and powerful solution for web crawling and data extraction tasks. Its asynchronous architecture and advanced features make it suitable for a wide range of applications, from simple web scraping to complex, multi-page data extraction scenarios.\n",
+        "\n",
+        "For more information and advanced usage, please visit the [Crawl4AI documentation](https://docs.crawl4ai.com/).\n",
+        "\n",
+        "Happy crawling!"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "venv",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.10.13"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

tests/browser/docker/test_docker_browser.py

@@ -9,6 +9,8 @@ import os
 import sys
 import shutil
 import uuid
+import json
+from typing import List, Dict, Any, Optional, Tuple
 
 # Add the project root to Python path if running directly
 if __name__ == "__main__":
@@ -17,9 +19,9 @@ if __name__ == "__main__":
 from crawl4ai.browser import BrowserManager
 from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
 from crawl4ai.async_logger import AsyncLogger
-from crawl4ai.browser import DockerConfig
-from crawl4ai.browser import DockerRegistry
-from crawl4ai.browser import DockerUtils
+from crawl4ai.browser.docker_config import DockerConfig
+from crawl4ai.browser.docker_registry import DockerRegistry
+from crawl4ai.browser.docker_utils import DockerUtils
 
 # Create a logger for clear terminal output
 logger = AsyncLogger(verbose=True, log_file=None)
@@ -136,7 +138,7 @@ async def test_docker_components():
             
         # Verify Chrome is installed in the container
         returncode, stdout, stderr = await docker_utils.exec_in_container(
-            container_id, ["which", "chromium"]
+            container_id, ["which", "google-chrome"]
         )
         
         if returncode != 0:
@@ -149,7 +151,7 @@ async def test_docker_components():
         
         # Test Chrome version
         returncode, stdout, stderr = await docker_utils.exec_in_container(
-            container_id, ["chromium", "--version"]
+            container_id, ["google-chrome", "--version"]
         )
         
         if returncode != 0:
@@ -530,7 +532,7 @@ async def test_docker_registry_reuse():
         logger.info("First browser started successfully", tag="TEST")
         
         # Get container ID from the strategy
-        docker_strategy1 = manager1.strategy
+        docker_strategy1 = manager1._strategy
         container_id1 = docker_strategy1.container_id
         logger.info(f"First browser container ID: {container_id1[:12]}", tag="TEST")
         
@@ -560,7 +562,7 @@ async def test_docker_registry_reuse():
         logger.info("Second browser started successfully", tag="TEST")
         
         # Get container ID from the second strategy
-        docker_strategy2 = manager2.strategy
+        docker_strategy2 = manager2._strategy
         container_id2 = docker_strategy2.container_id
         logger.info(f"Second browser container ID: {container_id2[:12]}", tag="TEST")
         
@@ -608,10 +610,10 @@ async def run_tests():
         return
     
     # First test Docker components
-    # setup_result = await test_docker_components()
-    # if not setup_result:
-    #     logger.error("Docker component tests failed - skipping browser tests", tag="TEST")
-    #     return
+    setup_result = await test_docker_components()
+    if not setup_result:
+        logger.error("Docker component tests failed - skipping browser tests", tag="TEST")
+        return
     
     # Run browser tests
     results.append(await test_docker_connect_mode())

tests/browser/manager/demo_browser_manager.py

@@ -1,525 +0,0 @@
-"""Demo script for testing the enhanced BrowserManager.
-
-This script demonstrates the browser pooling capabilities of the enhanced
-BrowserManager with various configurations and usage patterns.
-"""
-
-import asyncio
-import time
-import random
-
-from crawl4ai.browser.manager import BrowserManager, UnavailableBehavior
-from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
-from crawl4ai.async_logger import AsyncLogger
-
-import playwright
-
-SAFE_URLS = [
-    "https://example.com",
-    "https://example.com/page1",
-    "https://httpbin.org/get",
-    "https://httpbin.org/html",
-    "https://httpbin.org/ip",
-    "https://httpbin.org/user-agent",
-    "https://httpbin.org/headers",
-    "https://httpbin.org/cookies",
-    "https://httpstat.us/200",
-    "https://httpstat.us/301",
-    "https://httpstat.us/404",
-    "https://httpstat.us/500",
-    "https://jsonplaceholder.typicode.com/posts/1",
-    "https://jsonplaceholder.typicode.com/posts/2",
-    "https://jsonplaceholder.typicode.com/posts/3",
-    "https://jsonplaceholder.typicode.com/posts/4",
-    "https://jsonplaceholder.typicode.com/posts/5",
-    "https://jsonplaceholder.typicode.com/comments/1",
-    "https://jsonplaceholder.typicode.com/comments/2",
-    "https://jsonplaceholder.typicode.com/users/1",
-    "https://jsonplaceholder.typicode.com/users/2",
-    "https://jsonplaceholder.typicode.com/albums/1",
-    "https://jsonplaceholder.typicode.com/albums/2",
-    "https://jsonplaceholder.typicode.com/photos/1",
-    "https://jsonplaceholder.typicode.com/photos/2",
-    "https://jsonplaceholder.typicode.com/todos/1",
-    "https://jsonplaceholder.typicode.com/todos/2",
-    "https://www.iana.org",
-    "https://www.iana.org/domains",
-    "https://www.iana.org/numbers",
-    "https://www.iana.org/protocols",
-    "https://www.iana.org/about",
-    "https://www.iana.org/time-zones",
-    "https://www.data.gov",
-    "https://catalog.data.gov/dataset",
-    "https://www.archives.gov",
-    "https://www.usa.gov",
-    "https://www.loc.gov",
-    "https://www.irs.gov",
-    "https://www.census.gov",
-    "https://www.bls.gov",
-    "https://www.gpo.gov",
-    "https://www.w3.org",
-    "https://www.w3.org/standards",
-    "https://www.w3.org/WAI",
-    "https://www.rfc-editor.org",
-    "https://www.ietf.org",
-    "https://www.icann.org",
-    "https://www.internetsociety.org",
-    "https://www.python.org"
-]
-
-async def basic_pooling_demo():
-    """Demonstrate basic browser pooling functionality."""
-    print("\n=== Basic Browser Pooling Demo ===")
-    
-    # Create logger
-    logger = AsyncLogger(verbose=True)
-    
-    # Create browser configurations
-    config1 = BrowserConfig(
-        browser_type="chromium",
-        headless=True,
-        browser_mode="playwright"
-    )
-    
-    config2 = BrowserConfig(
-        browser_type="chromium", 
-        headless=True,
-        browser_mode="cdp"
-    )
-    
-    # Create browser manager with on-demand behavior
-    manager = BrowserManager(
-        browser_config=config1,
-        logger=logger,
-        unavailable_behavior=UnavailableBehavior.ON_DEMAND,
-        max_browsers_per_config=3
-    )
-    
-    try:
-        # Initialize pool with both configurations
-        print("Initializing browser pool...")
-        await manager.initialize_pool(
-            browser_configs=[config1, config2],
-            browsers_per_config=2
-        )
-        
-        # Display initial pool status
-        status = await manager.get_pool_status()
-        print(f"Initial pool status: {status}")
-        
-        # Create crawler run configurations
-        run_config1 = CrawlerRunConfig()
-        run_config2 = CrawlerRunConfig()
-        
-        # Simulate concurrent page requests
-        print("\nGetting pages for parallel crawling...")
-        
-        # Function to simulate crawling
-        async def simulate_crawl(index: int, config: BrowserConfig, run_config: CrawlerRunConfig):
-            print(f"Crawler {index}: Requesting page...")
-            page, context, strategy = await manager.get_page(run_config, config)
-            print(f"Crawler {index}: Got page, navigating to example.com...")
-            
-            try:
-                await page.goto("https://example.com")
-                title = await page.title()
-                print(f"Crawler {index}: Page title: {title}")
-                
-                # Simulate work
-                await asyncio.sleep(random.uniform(1, 3))
-                print(f"Crawler {index}: Work completed, releasing page...")
-                
-                # Check dynamic page content
-                content = await page.content()
-                content_length = len(content)
-                print(f"Crawler {index}: Page content length: {content_length}")
-                
-            except Exception as e:
-                print(f"Crawler {index}: Error: {str(e)}")
-            finally:
-                # Release the page
-                await manager.release_page(page, strategy, config)
-                print(f"Crawler {index}: Page released")
-        
-        # Create 5 parallel crawls
-        crawl_tasks = []
-        for i in range(5):
-            # Alternate between configurations
-            config = config1 if i % 2 == 0 else config2
-            run_config = run_config1 if i % 2 == 0 else run_config2
-            
-            task = asyncio.create_task(simulate_crawl(i+1, config, run_config))
-            crawl_tasks.append(task)
-        
-        # Wait for all crawls to complete
-        await asyncio.gather(*crawl_tasks)
-        
-        # Display final pool status
-        status = await manager.get_pool_status()
-        print(f"\nFinal pool status: {status}")
-        
-    finally:
-        # Clean up
-        print("\nClosing browser manager...")
-        await manager.close()
-        print("Browser manager closed")
-
-
-async def prewarm_pages_demo():
-    """Demonstrate page pre-warming functionality."""
-    print("\n=== Page Pre-warming Demo ===")
-    
-    # Create logger
-    logger = AsyncLogger(verbose=True)
-    
-    # Create browser configuration
-    config = BrowserConfig(
-        browser_type="chromium",
-        headless=True,
-        browser_mode="playwright"
-    )
-    
-    # Create crawler run configurations for pre-warming
-    run_config1 = CrawlerRunConfig(
-        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
-    )
-    
-    run_config2 = CrawlerRunConfig(
-        user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15"
-    )
-    
-    # Create page pre-warm configurations
-    page_configs = [
-        (config, run_config1, 2),  # 2 pages with run_config1
-        (config, run_config2, 3)   # 3 pages with run_config2
-    ]
-    
-    # Create browser manager
-    manager = BrowserManager(
-        browser_config=config,
-        logger=logger,
-        unavailable_behavior=UnavailableBehavior.EXCEPTION
-    )
-    
-    try:
-        # Initialize pool with pre-warmed pages
-        print("Initializing browser pool with pre-warmed pages...")
-        await manager.initialize_pool(
-            browser_configs=[config],
-            browsers_per_config=2,
-            page_configs=page_configs
-        )
-        
-        # Display pool status
-        status = await manager.get_pool_status()
-        print(f"Pool status after pre-warming: {status}")
-        
-        # Simulate using pre-warmed pages
-        print("\nUsing pre-warmed pages...")
-        
-        async def use_prewarm_page(index: int, run_config: CrawlerRunConfig):
-            print(f"Task {index}: Requesting pre-warmed page...")
-            page, context, strategy = await manager.get_page(run_config, config)
-            
-            try:
-                print(f"Task {index}: Got page, navigating to example.com...")
-                await page.goto("https://example.com")
-                
-                # Verify user agent was applied correctly
-                user_agent = await page.evaluate("() => navigator.userAgent")
-                print(f"Task {index}: User agent: {user_agent}")
-                
-                # Get page title
-                title = await page.title()
-                print(f"Task {index}: Page title: {title}")
-                
-                # Simulate work
-                await asyncio.sleep(1)
-            finally:
-                # Release the page
-                print(f"Task {index}: Releasing page...")
-                await manager.release_page(page, strategy, config)
-        
-        # Create tasks to use pre-warmed pages
-        tasks = []
-        # Use run_config1 pages
-        for i in range(2):
-            tasks.append(asyncio.create_task(use_prewarm_page(i+1, run_config1)))
-        
-        # Use run_config2 pages
-        for i in range(3):
-            tasks.append(asyncio.create_task(use_prewarm_page(i+3, run_config2)))
-        
-        # Wait for all tasks to complete
-        await asyncio.gather(*tasks)
-        
-        # Try to use more pages than we pre-warmed (should raise exception)
-        print("\nTrying to use more pages than pre-warmed...")
-        try:
-            page, context, strategy = await manager.get_page(run_config1, config)
-            try:
-                print("Got extra page (unexpected)")
-                await page.goto("https://example.com")
-            finally:
-                await manager.release_page(page, strategy, config)
-        except Exception as e:
-            print(f"Expected exception when requesting more pages: {str(e)}")
-        
-    finally:
-        # Clean up
-        print("\nClosing browser manager...")
-        await manager.close()
-        print("Browser manager closed")
-
-
-async def prewarm_on_demand_demo():
-    """Demonstrate pre-warming with on-demand browser creation."""
-    print("\n=== Pre-warming with On-Demand Browser Creation Demo ===")
-    
-    # Create logger
-    logger = AsyncLogger(verbose=True)
-    
-    # Create browser configuration
-    config = BrowserConfig(
-        browser_type="chromium",
-        headless=True,
-        browser_mode="playwright"
-    )
-    
-    # Create crawler run configurations
-    run_config = CrawlerRunConfig(
-        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
-    )
-    
-    # Create page pre-warm configurations - just pre-warm 2 pages
-    page_configs = [
-        (config, run_config, 2)
-    ]
-    
-    # Create browser manager with ON_DEMAND behavior
-    manager = BrowserManager(
-        browser_config=config,
-        logger=logger,
-        unavailable_behavior=UnavailableBehavior.ON_DEMAND,
-        max_browsers_per_config=5  # Allow up to 5 browsers
-    )
-    
-    try:
-        # Initialize pool with pre-warmed pages
-        print("Initializing browser pool with pre-warmed pages...")
-        await manager.initialize_pool(
-            browser_configs=[config],
-            browsers_per_config=1,  # Start with just 1 browser
-            page_configs=page_configs
-        )
-        
-        # Display initial pool status
-        status = await manager.get_pool_status()
-        print(f"Initial pool status: {status}")
-        
-        # Simulate using more pages than pre-warmed - should create browsers on demand
-        print("\nUsing more pages than pre-warmed (should create on demand)...")
-        
-        async def use_page(index: int):
-            print(f"Task {index}: Requesting page...")
-            page, context, strategy = await manager.get_page(run_config, config)
-            
-            try:
-                print(f"Task {index}: Got page, navigating to example.com...")
-                await page.goto("https://example.com")
-                
-                # Get page title
-                title = await page.title()
-                print(f"Task {index}: Page title: {title}")
-                
-                # Simulate work for a varying amount of time
-                work_time = 1 + (index * 0.5)  # Stagger completion times
-                print(f"Task {index}: Working for {work_time} seconds...")
-                await asyncio.sleep(work_time)
-                print(f"Task {index}: Work completed")
-            finally:
-                # Release the page
-                print(f"Task {index}: Releasing page...")
-                await manager.release_page(page, strategy, config)
-        
-        # Create more tasks than pre-warmed pages
-        tasks = []
-        for i in range(5):  # Try to use 5 pages when only 2 are pre-warmed
-            tasks.append(asyncio.create_task(use_page(i+1)))
-        
-        # Wait for all tasks to complete
-        await asyncio.gather(*tasks)
-        
-        # Display final pool status - should show on-demand created browsers
-        status = await manager.get_pool_status()
-        print(f"\nFinal pool status: {status}")
-        
-    finally:
-        # Clean up
-        print("\nClosing browser manager...")
-        await manager.close()
-        print("Browser manager closed")
-
-
-async def high_volume_demo():
-    """Demonstrate high-volume access to pre-warmed pages."""
-    print("\n=== High Volume Pre-warmed Pages Demo ===")
-    
-    # Create logger
-    logger = AsyncLogger(verbose=True)
-    
-    # Create browser configuration
-    config = BrowserConfig(
-        browser_type="chromium",
-        headless=True,
-        browser_mode="playwright"
-    )
-    
-    # Create crawler run configuration
-    run_config = CrawlerRunConfig(
-        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
-    )
-    
-    # Set up dimensions
-    browser_count = 10
-    pages_per_browser = 5
-    total_pages = browser_count * pages_per_browser
-    
-    # Create page pre-warm configuration
-    page_configs = [
-        (config, run_config, total_pages)
-    ]
-    
-    print(f"Preparing {browser_count} browsers with {pages_per_browser} pages each ({total_pages} total pages)")
-    
-    # Create browser manager with ON_DEMAND behavior as fallback
-    # No need to specify max_browsers_per_config as it will be calculated automatically
-    manager = BrowserManager(
-        browser_config=config,
-        logger=logger,
-        unavailable_behavior=UnavailableBehavior.ON_DEMAND
-    )
-    
-    try:
-        # Initialize pool with browsers and pre-warmed pages
-        print(f"Pre-warming {total_pages} pages...")
-        start_time = time.time()
-        await manager.initialize_pool(
-            browser_configs=[config],
-            browsers_per_config=browser_count,
-            page_configs=page_configs
-        )
-        warmup_time = time.time() - start_time
-        print(f"Pre-warming completed in {warmup_time:.2f} seconds")
-        
-        # Display pool status
-        status = await manager.get_pool_status()
-        print(f"Pool status after pre-warming: {status}")
-        
-        # Simulate using all pre-warmed pages simultaneously
-        print(f"\nSending {total_pages} crawl requests simultaneously...")
-        
-        async def crawl_page(index: int):
-            # url = f"https://example.com/page{index}"
-            url = SAFE_URLS[index % len(SAFE_URLS)]
-            print(f"Page {index}: Requesting page...")            
-            # Measure time to acquire page
-            page_start = time.time()
-            page, context, strategy = await manager.get_page(run_config, config)
-            page_acquisition_time = time.time() - page_start
-            
-            try:
-                # Navigate to the URL
-                nav_start = time.time()
-                await page.goto(url, timeout=5000)
-                navigation_time = time.time() - nav_start
-                
-                # Get the page title
-                title = await page.title()
-                
-                return {
-                    "index": index,
-                    "url": url,
-                    "title": title,
-                    "page_acquisition_time": page_acquisition_time,
-                    "navigation_time": navigation_time
-                }
-            except playwright._impl._errors.TimeoutError as e:
-                # print(f"Page {index}: Navigation timed out - {e}")
-                return {
-                    "index": index,
-                    "url": url,
-                    "title": "Navigation timed out",
-                    "page_acquisition_time": page_acquisition_time,
-                    "navigation_time": 0
-                }
-            finally:
-                # Release the page
-                await manager.release_page(page, strategy, config)
-        
-        # Create and execute all tasks simultaneously
-        start_time = time.time()
-
-        # Non-parallel way
-        # for i in range(total_pages):
-        #     await crawl_page(i+1)
-
-        tasks = [crawl_page(i+1) for i in range(total_pages)]
-        results = await asyncio.gather(*tasks)
-        total_time = time.time() - start_time
-        
-        # # Print all titles
-        # for result in results:
-        #     print(f"Page {result['index']} ({result['url']}): Title: {result['title']}")
-        #     print(f"  Page acquisition time: {result['page_acquisition_time']:.4f}s")
-        #     print(f"  Navigation time: {result['navigation_time']:.4f}s")
-        #     print(f"  Total time: {result['page_acquisition_time'] + result['navigation_time']:.4f}s")
-        #     print("-" * 40)
-        
-        # Report results
-        print(f"\nAll {total_pages} crawls completed in {total_time:.2f} seconds")
-        
-        # Calculate statistics
-        acquisition_times = [r["page_acquisition_time"] for r in results]
-        navigation_times = [r["navigation_time"] for r in results]
-        
-        avg_acquisition = sum(acquisition_times) / len(acquisition_times)
-        max_acquisition = max(acquisition_times)
-        min_acquisition = min(acquisition_times)
-        
-        avg_navigation = sum(navigation_times) / len(navigation_times)
-        max_navigation = max(navigation_times)
-        min_navigation = min(navigation_times)
-        
-        print("\nPage acquisition times:")
-        print(f"  Average: {avg_acquisition:.4f}s")
-        print(f"  Min: {min_acquisition:.4f}s")
-        print(f"  Max: {max_acquisition:.4f}s")
-        
-        print("\nPage navigation times:")
-        print(f"  Average: {avg_navigation:.4f}s")
-        print(f"  Min: {min_navigation:.4f}s")
-        print(f"  Max: {max_navigation:.4f}s")
-        
-        # Display final pool status
-        status = await manager.get_pool_status()
-        print(f"\nFinal pool status: {status}")
-        
-    finally:
-        # Clean up
-        print("\nClosing browser manager...")
-        await manager.close()
-        print("Browser manager closed")
-
-
-async def main():
-    """Run all demos."""
-    # await basic_pooling_demo()
-    # await prewarm_pages_demo()
-    # await prewarm_on_demand_demo()
-    await high_volume_demo()
-    # Additional demo functions can be added here
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

tests/browser/test_builtin_browser.py

@@ -56,13 +56,13 @@ async def test_builtin_browser_creation():
 
     # Step 2: Check if we have a BuiltinBrowserStrategy
     print(f"\n{INFO}2. Checking if we have a BuiltinBrowserStrategy{RESET}")
-    if isinstance(manager.strategy, BuiltinBrowserStrategy):
+    if isinstance(manager._strategy, BuiltinBrowserStrategy):
         print(
-            f"{SUCCESS}Correct strategy type: {manager.strategy.__class__.__name__}{RESET}"
+            f"{SUCCESS}Correct strategy type: {manager._strategy.__class__.__name__}{RESET}"
         )
     else:
         print(
-            f"{ERROR}Wrong strategy type: {manager.strategy.__class__.__name__}{RESET}"
+            f"{ERROR}Wrong strategy type: {manager._strategy.__class__.__name__}{RESET}"
         )
         return None
 
@@ -77,7 +77,7 @@ async def test_builtin_browser_creation():
 
     # Step 4: Get browser info from the strategy
     print(f"\n{INFO}4. Getting browser information{RESET}")
-    browser_info = manager.strategy.get_browser_info()
+    browser_info = manager._strategy.get_builtin_browser_info()
     if browser_info:
         print(f"{SUCCESS}Browser info retrieved:{RESET}")
         for key, value in browser_info.items():
@@ -149,7 +149,7 @@ async def test_browser_status_management(manager: BrowserManager):
     # Step 1: Get browser status
     print(f"\n{INFO}1. Getting browser status{RESET}")
     try:
-        status = await manager.strategy.get_builtin_browser_status()
+        status = await manager._strategy.get_builtin_browser_status()
         print(f"{SUCCESS}Browser status:{RESET}")
         print(f"  Running: {status['running']}")
         print(f"  CDP URL: {status['cdp_url']}")
@@ -160,7 +160,7 @@ async def test_browser_status_management(manager: BrowserManager):
     # Step 2: Test killing the browser
     print(f"\n{INFO}2. Testing killing the browser{RESET}")
     try:
-        result = await manager.strategy.kill_builtin_browser()
+        result = await manager._strategy.kill_builtin_browser()
         if result:
             print(f"{SUCCESS}Browser killed successfully{RESET}")
         else:
@@ -172,7 +172,7 @@ async def test_browser_status_management(manager: BrowserManager):
     # Step 3: Check status after kill
     print(f"\n{INFO}3. Checking status after kill{RESET}")
     try:
-        status = await manager.strategy.get_builtin_browser_status()
+        status = await manager._strategy.get_builtin_browser_status()
         if not status["running"]:
             print(f"{SUCCESS}Browser is correctly reported as not running{RESET}")
         else:
@@ -184,7 +184,7 @@ async def test_browser_status_management(manager: BrowserManager):
     # Step 4: Launch a new browser
     print(f"\n{INFO}4. Launching a new browser{RESET}")
     try:
-        cdp_url = await manager.strategy.launch_builtin_browser(
+        cdp_url = await manager._strategy.launch_builtin_browser(
             browser_type="chromium", headless=True
         )
         if cdp_url:
@@ -205,7 +205,7 @@ async def test_multiple_managers():
 
     # Step 1: Create first manager
     print(f"\n{INFO}1. Creating first browser manager{RESET}")
-    browser_config1 = BrowserConfig(browser_mode="builtin", headless=True)
+    browser_config1 = (BrowserConfig(browser_mode="builtin", headless=True),)
     manager1 = BrowserManager(browser_config=browser_config1, logger=logger)
 
     # Step 2: Create second manager
@@ -223,8 +223,8 @@ async def test_multiple_managers():
         print(f"{SUCCESS}Second manager started{RESET}")
 
         # Check if they got the same CDP URL
-        cdp_url1 = manager1.strategy.config.cdp_url
-        cdp_url2 = manager2.strategy.config.cdp_url
+        cdp_url1 = manager1._strategy.config.cdp_url
+        cdp_url2 = manager2._strategy.config.cdp_url
 
         if cdp_url1 == cdp_url2:
             print(
@@ -316,7 +316,7 @@ async def test_edge_cases():
 
         # Kill the browser directly
         print(f"{INFO}Killing the browser...{RESET}")
-        await manager.strategy.kill_builtin_browser()
+        await manager._strategy.kill_builtin_browser()
         print(f"{SUCCESS}Browser killed{RESET}")
 
         # Try to get a page (should fail or launch a new browser)
@@ -350,7 +350,7 @@ async def cleanup_browsers():
 
     try:
         # No need to start, just access the strategy directly
-        strategy = manager.strategy
+        strategy = manager._strategy
         if isinstance(strategy, BuiltinBrowserStrategy):
             result = await strategy.kill_builtin_browser()
             if result:
@@ -420,7 +420,7 @@ async def test_performance_scaling():
             user_data_dir=os.path.join(temp_dir, f"browser_profile_{i}"),
         )
         manager = BrowserManager(browser_config=browser_config, logger=logger)
-        manager.strategy.shutting_down = True
+        manager._strategy.shutting_down = True
         manager_configs.append((manager, i, port))
 
     # Define async function to start a single manager
@@ -614,7 +614,7 @@ async def test_performance_scaling_lab( num_browsers: int = 10, pages_per_browse
             user_data_dir=os.path.join(temp_dir, f"browser_profile_{i}"),
         )
         manager = BrowserManager(browser_config=browser_config, logger=logger)
-        manager.strategy.shutting_down = True
+        manager._strategy.shutting_down = True
         manager_configs.append((manager, i, port))
 
     # Define async function to start a single manager
@@ -781,16 +781,15 @@ async def main():
         # await manager.close()
 
         # Run multiple managers test
-        await test_multiple_managers()
+        # await test_multiple_managers()
 
         # Run performance scaling test
         await test_performance_scaling()
-
         # Run cleanup test
-        await cleanup_browsers()
+        # await cleanup_browsers()
 
         # Run edge cases test
-        await test_edge_cases()
+        # await test_edge_cases()
 
         print(f"\n{SUCCESS}All tests completed!{RESET}")
 

tests/browser/test_cdp_strategy.py

@@ -25,7 +25,6 @@ async def test_cdp_launch_connect():
     
     browser_config = BrowserConfig(
         use_managed_browser=True,
-        browser_mode="cdp",
         headless=True
     )
     
@@ -71,8 +70,8 @@ async def test_cdp_with_user_data_dir():
     logger.info(f"Created temporary user data directory: {user_data_dir}", tag="TEST")
     
     browser_config = BrowserConfig(
+        use_managed_browser=True,
         headless=True,
-        browser_mode="cdp",
         user_data_dir=user_data_dir
     )
     
@@ -211,7 +210,7 @@ async def run_tests():
     results = []
     
     # results.append(await test_cdp_launch_connect())
-    results.append(await test_cdp_with_user_data_dir())
+    # results.append(await test_cdp_with_user_data_dir())
     results.append(await test_cdp_session_management())
     
     # Print summary

tests/browser/test_playwright_strategy.py

@@ -6,7 +6,6 @@ and serve as functional tests.
 
 import asyncio
 import os
-import re
 import sys
 
 # Add the project root to Python path if running directly
@@ -20,53 +19,6 @@ from crawl4ai.async_logger import AsyncLogger
 # Create a logger for clear terminal output
 logger = AsyncLogger(verbose=True, log_file=None)
 
-
-
-async def test_start_close():
-    # Create browser config for standard Playwright
-    browser_config = BrowserConfig(
-        headless=True,
-        viewport_width=1280,
-        viewport_height=800
-    )
-    
-    # Create browser manager with the config
-    manager = BrowserManager(browser_config=browser_config, logger=logger)
-    
-    try:
-        for _ in range(4):
-            # Start the browser
-            await manager.start()
-            logger.info("Browser started successfully", tag="TEST")
-
-            # Get a page
-            page, context = await manager.get_page(CrawlerRunConfig())
-            logger.info("Got page successfully", tag="TEST")
-            
-            # Navigate to a website
-            await page.goto("https://example.com")
-            logger.info("Navigated to example.com", tag="TEST")
-            
-            # Get page title
-            title = await page.title()
-            logger.info(f"Page title: {title}", tag="TEST")
-            
-            # Clean up
-            await manager.close()
-            logger.info("Browser closed successfully", tag="TEST")
-   
-            await asyncio.sleep(1)  # Wait for a moment before restarting
-
-    except Exception as e:
-        logger.error(f"Test failed: {str(e)}", tag="TEST")
-        # Ensure cleanup
-        try:
-            await manager.close()
-        except:
-            pass
-        return False
-    return True
-
 async def test_playwright_basic():
     """Test basic Playwright browser functionality."""
     logger.info("Testing standard Playwright browser", tag="TEST")
@@ -296,10 +248,9 @@ async def run_tests():
     """Run all tests sequentially."""
     results = []
     
-    # results.append(await test_start_close())
-    # results.append(await test_playwright_basic())
-    # results.append(await test_playwright_text_mode())
-    # results.append(await test_playwright_context_reuse())
+    results.append(await test_playwright_basic())
+    results.append(await test_playwright_text_mode())
+    results.append(await test_playwright_context_reuse())
     results.append(await test_playwright_session_management())
     
     # Print summary

tests/pipeline/demo_browser_hub_pipeline.py

@@ -1,222 +0,0 @@
-# demo_browser_hub.py
-
-import asyncio
-from typing import List
-
-from crawl4ai.browser.browser_hub import BrowserHub
-from pipeline import create_pipeline
-from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
-from crawl4ai.async_logger import AsyncLogger
-from crawl4ai.models import CrawlResultContainer
-from crawl4ai.cache_context import CacheMode
-from crawl4ai import DefaultMarkdownGenerator
-from crawl4ai import PruningContentFilter
-
-async def create_prewarmed_browser_hub(urls_to_crawl: List[str]):
-    """Create a pre-warmed browser hub with 10 browsers and 5 pages each."""
-    # Set up logging
-    logger = AsyncLogger(verbose=True)
-    logger.info("Setting up pre-warmed browser hub", tag="DEMO")
-    
-    # Create browser configuration
-    browser_config = BrowserConfig(
-        browser_type="chromium",
-        headless=True,  # Set to False to see the browsers in action
-        viewport_width=1280,
-        viewport_height=800,
-        light_mode=True,  # Optimize for performance
-        java_script_enabled=True
-    )
-    
-    # Create crawler configurations for pre-warming with different user agents
-    # This allows pages to be ready for different scenarios
-    crawler_configs = [
-        CrawlerRunConfig(
-            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-            wait_until="networkidle"
-        ),
-        # CrawlerRunConfig(
-        #     user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15",
-        #     wait_until="networkidle"
-        # ),
-        # CrawlerRunConfig(
-        #     user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36",
-        #     wait_until="networkidle"
-        # )
-    ]
-    
-    # Number of browsers and pages per browser
-    num_browsers = 1
-    pages_per_browser = 1
-    
-    # Distribute pages across configurations
-    # We'll create a total of 50 pages (10 browsers × 5 pages)
-    page_configs = []
-    total_pages = num_browsers * pages_per_browser
-    pages_per_config = total_pages // len(crawler_configs)
-    
-    for i, config in enumerate(crawler_configs):
-        # For the last config, add any remaining pages
-        if i == len(crawler_configs) - 1:
-            remaining = total_pages - (pages_per_config * (len(crawler_configs) - 1))
-            page_configs.append((browser_config, config, remaining))
-        else:
-            page_configs.append((browser_config, config, pages_per_config))
-    
-    # Create browser hub with pre-warmed pages
-    start_time = asyncio.get_event_loop().time()
-    logger.info("Initializing browser hub with pre-warmed pages...", tag="DEMO")
-    
-    hub = await BrowserHub.get_browser_manager(
-        config=browser_config,
-        hub_id="demo_hub",
-        logger=logger,
-        max_browsers_per_config=num_browsers,
-        max_pages_per_browser=pages_per_browser,
-        initial_pool_size=num_browsers,
-        page_configs=page_configs
-    )
-    
-    end_time = asyncio.get_event_loop().time()
-    logger.success(
-        message="Browser hub initialized with {total_pages} pre-warmed pages in {duration:.2f} seconds",
-        tag="DEMO",
-        params={
-            "total_pages": total_pages,
-            "duration": end_time - start_time
-        }
-    )
-    
-    # Get and display pool status
-    status = await hub.get_pool_status()
-    logger.info(
-        message="Browser pool status: {status}",
-        tag="DEMO",
-        params={"status": status}
-    )
-    
-    return hub
-
-async def crawl_urls_with_hub(hub, urls: List[str]) -> List[CrawlResultContainer]:
-    """Crawl a list of URLs using a pre-warmed browser hub."""
-    logger = AsyncLogger(verbose=True)
-    
-    # Create crawler configuration
-    crawler_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-        markdown_generator=DefaultMarkdownGenerator(
-            content_filter=PruningContentFilter(
-                threshold=0.48,
-                threshold_type="fixed",
-                min_word_threshold=0
-            )
-        ),
-        wait_until="networkidle",
-        screenshot=True
-    )
-    
-    # Create pipeline with the browser hub
-    pipeline = await create_pipeline(
-        browser_hub=hub,
-        logger=logger
-    )
-    
-    results = []
-    
-    # Crawl all URLs in parallel
-    async def crawl_url(url):
-        logger.info(f"Crawling {url}...", tag="CRAWL")
-        result = await pipeline.crawl(url=url, config=crawler_config)
-        logger.success(f"Completed crawl of {url}", tag="CRAWL")
-        return result
-    
-    # Create tasks for all URLs
-    tasks = [crawl_url(url) for url in urls]
-    
-    # Execute all tasks in parallel and collect results
-    results = await asyncio.gather(*tasks)
-    
-    return results
-
-async def main():
-    """Main demo function."""
-    # List of URLs to crawl
-    urls_to_crawl = [
-        "https://example.com",
-        # "https://www.python.org",
-        # "https://httpbin.org/html",
-        # "https://news.ycombinator.com",
-        # "https://github.com",
-        # "https://pypi.org",
-        # "https://docs.python.org/3/",
-        # "https://opensource.org",
-        # "https://whatismyipaddress.com",
-        # "https://en.wikipedia.org/wiki/Web_scraping"
-    ]
-    
-    # Set up logging
-    logger = AsyncLogger(verbose=True)
-    logger.info("Starting browser hub demo", tag="DEMO")
-    
-    try:
-        # Create pre-warmed browser hub
-        hub = await create_prewarmed_browser_hub(urls_to_crawl)
-        
-        # Use hub to crawl URLs
-        logger.info("Crawling URLs in parallel...", tag="DEMO")
-        start_time = asyncio.get_event_loop().time()
-        
-        results = await crawl_urls_with_hub(hub, urls_to_crawl)
-        
-        end_time = asyncio.get_event_loop().time()
-        
-        # Display results
-        logger.success(
-            message="Crawled {count} URLs in {duration:.2f} seconds (average: {avg:.2f} seconds per URL)",
-            tag="DEMO",
-            params={
-                "count": len(results),
-                "duration": end_time - start_time,
-                "avg": (end_time - start_time) / len(results)
-            }
-        )
-        
-        # Print summary of results
-        logger.info("Crawl results summary:", tag="DEMO")
-        for i, result in enumerate(results):
-            logger.info(
-                message="{idx}. {url}: Success={success}, Content length={length}",
-                tag="RESULT",
-                params={
-                    "idx": i+1,
-                    "url": result.url,
-                    "success": result.success,
-                    "length": len(result.html) if result.html else 0
-                }
-            )
-            
-            if result.success and result.markdown and result.markdown.raw_markdown:
-                # Print a snippet of the markdown
-                markdown_snippet = result.markdown.raw_markdown[:150] + "..."
-                logger.info(
-                    message="   Markdown: {snippet}",
-                    tag="RESULT",
-                    params={"snippet": markdown_snippet}
-                )
-        
-        # Display final browser pool status
-        status = await hub.get_pool_status()
-        logger.info(
-            message="Final browser pool status: {status}",
-            tag="DEMO",
-            params={"status": status}
-        )
-        
-    finally:
-        # Clean up
-        logger.info("Shutting down browser hub...", tag="DEMO")
-        await BrowserHub.shutdown_all()
-        logger.success("Demo completed", tag="DEMO")
-
-if __name__ == "__main__":
-    asyncio.run(main())

tests/pipeline/extended_browser_hub_tests.py

@@ -1,505 +0,0 @@
-# extended_browser_hub_tests.py
-
-import asyncio
-
-from crawl4ai.browser.browser_hub import BrowserHub
-from pipeline import create_pipeline
-from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
-from crawl4ai.async_logger import AsyncLogger
-from crawl4ai.cache_context import CacheMode
-
-# Common test URLs
-TEST_URLS = [
-    "https://example.com",
-    "https://example.com/page1",
-    "https://httpbin.org/html",
-    "https://httpbin.org/headers",
-    "https://httpbin.org/ip",
-    "https://httpstat.us/200"
-]
-
-class TestResults:
-    """Simple container for test results"""
-    def __init__(self, name: str):
-        self.name = name
-        self.results = []
-        self.start_time = None
-        self.end_time = None
-        self.errors = []
-    
-    @property
-    def duration(self) -> float:
-        if self.start_time and self.end_time:
-            return self.end_time - self.start_time
-        return 0
-    
-    @property
-    def success_rate(self) -> float:
-        if not self.results:
-            return 0
-        return sum(1 for r in self.results if r.success) / len(self.results) * 100
-
-    def log_summary(self, logger: AsyncLogger):
-        logger.info(f"=== Test: {self.name} ===", tag="SUMMARY")
-        logger.info(
-            message="Duration: {duration:.2f}s, Success rate: {success_rate:.1f}%, Results: {count}",
-            tag="SUMMARY",
-            params={
-                "duration": self.duration,
-                "success_rate": self.success_rate,
-                "count": len(self.results)
-            }
-        )
-        
-        if self.errors:
-            logger.error(
-                message="Errors ({count}): {errors}",
-                tag="SUMMARY",
-                params={
-                    "count": len(self.errors),
-                    "errors": "; ".join(str(e) for e in self.errors)
-                }
-            )
-
-# ======== TEST SCENARIO 1: Simple default configuration ========
-async def test_default_configuration():
-    """
-    Test Scenario 1: Simple default configuration
-    
-    This tests the basic case where the user does not provide any specific
-    browser configuration, relying on default auto-setup.
-    """
-    logger = AsyncLogger(verbose=True)
-    results = TestResults("Default Configuration")
-    
-    try:
-        # Create pipeline with no browser config
-        pipeline = await create_pipeline(logger=logger)
-        
-        # Start timing
-        results.start_time = asyncio.get_event_loop().time()
-        
-        # Create basic crawler config
-        crawler_config = CrawlerRunConfig(
-            cache_mode=CacheMode.BYPASS,
-            wait_until="domcontentloaded"
-        )
-        
-        # Process each URL sequentially
-        for url in TEST_URLS:
-            try:
-                logger.info(f"Crawling {url} with default configuration", tag="TEST")
-                result = await pipeline.crawl(url=url, config=crawler_config)
-                results.results.append(result)
-                
-                logger.success(
-                    message="Result: url={url}, success={success}, content_length={length}",
-                    tag="TEST",
-                    params={
-                        "url": url,
-                        "success": result.success,
-                        "length": len(result.html) if result.html else 0
-                    }
-                )
-            except Exception as e:
-                logger.error(f"Error crawling {url}: {str(e)}", tag="TEST")
-                results.errors.append(e)
-        
-        # End timing
-        results.end_time = asyncio.get_event_loop().time()
-        
-    except Exception as e:
-        logger.error(f"Test failed with error: {str(e)}", tag="TEST")
-        results.errors.append(e)
-    
-    # Log summary
-    results.log_summary(logger)
-    
-    return results
-
-# ======== TEST SCENARIO 2: Detailed custom configuration ========
-async def test_custom_configuration():
-    """
-    Test Scenario 2: Detailed custom configuration
-    
-    This tests the case where the user provides detailed browser configuration
-    to customize the browser behavior.
-    """
-    logger = AsyncLogger(verbose=True)
-    results = TestResults("Custom Configuration")
-    
-    try:
-        # Create custom browser config
-        browser_config = BrowserConfig(
-            browser_type="chromium",
-            headless=True,
-            viewport_width=1920,
-            viewport_height=1080,
-            user_agent="Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36",
-            light_mode=True,
-            ignore_https_errors=True,
-            extra_args=["--disable-extensions"]
-        )
-        
-        # Create custom crawler config
-        crawler_config = CrawlerRunConfig(
-            cache_mode=CacheMode.BYPASS,
-            wait_until="networkidle",
-            page_timeout=30000,
-            screenshot=True,
-            pdf=False,
-            screenshot_wait_for=0.5,
-            wait_for_images=True,
-            scan_full_page=True,
-            scroll_delay=0.2,
-            process_iframes=True,
-            remove_overlay_elements=True
-        )
-        
-        # Create pipeline with custom configuration
-        pipeline = await create_pipeline(
-            browser_config=browser_config,
-            logger=logger
-        )
-        
-        # Start timing
-        results.start_time = asyncio.get_event_loop().time()
-        
-        # Process each URL sequentially
-        for url in TEST_URLS:
-            try:
-                logger.info(f"Crawling {url} with custom configuration", tag="TEST")
-                result = await pipeline.crawl(url=url, config=crawler_config)
-                results.results.append(result)
-                
-                has_screenshot = result.screenshot is not None
-                
-                logger.success(
-                    message="Result: url={url}, success={success}, screenshot={screenshot}, content_length={length}",
-                    tag="TEST",
-                    params={
-                        "url": url,
-                        "success": result.success,
-                        "screenshot": has_screenshot,
-                        "length": len(result.html) if result.html else 0
-                    }
-                )
-            except Exception as e:
-                logger.error(f"Error crawling {url}: {str(e)}", tag="TEST")
-                results.errors.append(e)
-        
-        # End timing
-        results.end_time = asyncio.get_event_loop().time()
-        
-        # Get browser hub status from context
-        try:
-            # Run a dummy crawl to get the context with browser hub
-            context = await pipeline.process({"url": "about:blank", "config": crawler_config})
-            browser_hub = context.get("browser_hub")
-            if browser_hub:
-                status = await browser_hub.get_pool_status()
-                logger.info(
-                    message="Browser hub status: {status}",
-                    tag="TEST",
-                    params={"status": status}
-                )
-        except Exception as e:
-            logger.error(f"Failed to get browser hub status: {str(e)}", tag="TEST")
-    
-    except Exception as e:
-        logger.error(f"Test failed with error: {str(e)}", tag="TEST")
-        results.errors.append(e)
-    
-    # Log summary
-    results.log_summary(logger)
-    
-    return results
-
-# ======== TEST SCENARIO 3: Using pre-initialized browser hub ========
-async def test_preinitalized_browser_hub():
-    """
-    Test Scenario 3: Using pre-initialized browser hub
-    
-    This tests the case where a browser hub is initialized separately
-    and then passed to the pipeline.
-    """
-    logger = AsyncLogger(verbose=True)
-    results = TestResults("Pre-initialized Browser Hub")
-    
-    browser_hub = None
-    try:
-        # Create and initialize browser hub separately
-        logger.info("Initializing browser hub separately", tag="TEST")
-        
-        browser_config = BrowserConfig(
-            browser_type="chromium",
-            headless=True,
-            verbose=True
-        )
-        
-        browser_hub = await BrowserHub.get_browser_manager(
-            config=browser_config,
-            hub_id="test_preinitalized",
-            logger=logger,
-            max_browsers_per_config=2,
-            max_pages_per_browser=3,
-            initial_pool_size=2
-        )
-        
-        # Display initial status
-        status = await browser_hub.get_pool_status()
-        logger.info(
-            message="Initial browser hub status: {status}",
-            tag="TEST",
-            params={"status": status}
-        )
-        
-        # Create pipeline with pre-initialized browser hub
-        pipeline = await create_pipeline(
-            browser_hub=browser_hub,
-            logger=logger
-        )
-        
-        # Create crawler config
-        crawler_config = CrawlerRunConfig(
-            cache_mode=CacheMode.BYPASS,
-            wait_until="networkidle",
-            screenshot=True
-        )
-        
-        # Start timing
-        results.start_time = asyncio.get_event_loop().time()
-        
-        # Process URLs in parallel
-        async def crawl_url(url):
-            try:
-                logger.info(f"Crawling {url} with pre-initialized hub", tag="TEST")
-                result = await pipeline.crawl(url=url, config=crawler_config)
-                logger.success(f"Completed crawl of {url}", tag="TEST")
-                return result
-            except Exception as e:
-                logger.error(f"Error crawling {url}: {str(e)}", tag="TEST")
-                results.errors.append(e)
-                return None
-        
-        # Create tasks for all URLs
-        tasks = [crawl_url(url) for url in TEST_URLS]
-        
-        # Execute all tasks in parallel and collect results
-        all_results = await asyncio.gather(*tasks)
-        results.results = [r for r in all_results if r is not None]
-        
-        # End timing
-        results.end_time = asyncio.get_event_loop().time()
-        
-        # Display final status
-        status = await browser_hub.get_pool_status()
-        logger.info(
-            message="Final browser hub status: {status}",
-            tag="TEST",
-            params={"status": status}
-        )
-        
-    except Exception as e:
-        logger.error(f"Test failed with error: {str(e)}", tag="TEST")
-        results.errors.append(e)
-    
-    # Log summary
-    results.log_summary(logger)
-    
-    return results, browser_hub
-
-# ======== TEST SCENARIO 4: Parallel pipelines sharing browser hub ========
-async def test_parallel_pipelines():
-    """
-    Test Scenario 4: Multiple parallel pipelines sharing browser hub
-    
-    This tests the case where multiple pipelines share the same browser hub,
-    demonstrating resource sharing and parallel operation.
-    """
-    logger = AsyncLogger(verbose=True)
-    results = TestResults("Parallel Pipelines")
-    
-    # We'll reuse the browser hub from the previous test
-    _, browser_hub = await test_preinitalized_browser_hub()
-    
-    try:
-        # Create 3 pipelines that all share the same browser hub
-        pipelines = []
-        for i in range(3):
-            pipeline = await create_pipeline(
-                browser_hub=browser_hub,
-                logger=logger
-            )
-            pipelines.append(pipeline)
-        
-        logger.info(f"Created {len(pipelines)} pipelines sharing the same browser hub", tag="TEST")
-        
-        # Create crawler configs with different settings
-        configs = [
-            CrawlerRunConfig(wait_until="domcontentloaded", screenshot=False),
-            CrawlerRunConfig(wait_until="networkidle", screenshot=True),
-            CrawlerRunConfig(wait_until="load", scan_full_page=True)
-        ]
-        
-        # Start timing
-        results.start_time = asyncio.get_event_loop().time()
-        
-        # Function to process URLs with a specific pipeline
-        async def process_with_pipeline(pipeline_idx, urls):
-            pipeline_results = []
-            for url in urls:
-                try:
-                    logger.info(f"Pipeline {pipeline_idx} crawling {url}", tag="TEST")
-                    result = await pipelines[pipeline_idx].crawl(
-                        url=url, 
-                        config=configs[pipeline_idx]
-                    )
-                    pipeline_results.append(result)
-                    logger.success(
-                        message="Pipeline {idx} completed: url={url}, success={success}",
-                        tag="TEST",
-                        params={
-                            "idx": pipeline_idx,
-                            "url": url,
-                            "success": result.success
-                        }
-                    )
-                except Exception as e:
-                    logger.error(
-                        message="Pipeline {idx} error: {error}",
-                        tag="TEST",
-                        params={
-                            "idx": pipeline_idx,
-                            "error": str(e)
-                        }
-                    )
-                    results.errors.append(e)
-            return pipeline_results
-        
-        # Distribute URLs among pipelines
-        pipeline_urls = [
-            TEST_URLS[:2],
-            TEST_URLS[2:4],
-            TEST_URLS[4:5] * 2  # Duplicate the last URL to have 2 for pipeline 3
-        ]
-        
-        # Execute all pipelines in parallel
-        tasks = [
-            process_with_pipeline(i, urls) 
-            for i, urls in enumerate(pipeline_urls)
-        ]
-        
-        pipeline_results = await asyncio.gather(*tasks)
-        
-        # Flatten results
-        for res_list in pipeline_results:
-            results.results.extend(res_list)
-        
-        # End timing
-        results.end_time = asyncio.get_event_loop().time()
-        
-        # Display browser hub status
-        status = await browser_hub.get_pool_status()
-        logger.info(
-            message="Browser hub status after parallel pipelines: {status}",
-            tag="TEST",
-            params={"status": status}
-        )
-        
-    except Exception as e:
-        logger.error(f"Test failed with error: {str(e)}", tag="TEST")
-        results.errors.append(e)
-    
-    # Log summary
-    results.log_summary(logger)
-    
-    return results
-
-# ======== TEST SCENARIO 5: Browser hub with connection string ========
-async def test_connection_string():
-    """
-    Test Scenario 5: Browser hub with connection string
-    
-    This tests the case where a browser hub is initialized from a connection string,
-    simulating connecting to a running browser hub service.
-    """
-    logger = AsyncLogger(verbose=True)
-    results = TestResults("Connection String")
-    
-    try:
-        # Create pipeline with connection string
-        # Note: In a real implementation, this would connect to an existing service
-        # For this test, we're using a simulated connection
-        connection_string = "localhost:9222"  # Simulated connection string
-        
-        pipeline = await create_pipeline(
-            browser_hub_connection=connection_string,
-            logger=logger
-        )
-        
-        # Create crawler config
-        crawler_config = CrawlerRunConfig(
-            cache_mode=CacheMode.BYPASS,
-            wait_until="networkidle"
-        )
-        
-        # Start timing
-        results.start_time = asyncio.get_event_loop().time()
-        
-        # Test with a single URL
-        url = TEST_URLS[0]
-        try:
-            logger.info(f"Crawling {url} with connection string hub", tag="TEST")
-            result = await pipeline.crawl(url=url, config=crawler_config)
-            results.results.append(result)
-            
-            logger.success(
-                message="Result: url={url}, success={success}, content_length={length}",
-                tag="TEST",
-                params={
-                    "url": url,
-                    "success": result.success,
-                    "length": len(result.html) if result.html else 0
-                }
-            )
-        except Exception as e:
-            logger.error(f"Error crawling {url}: {str(e)}", tag="TEST")
-            results.errors.append(e)
-        
-        # End timing
-        results.end_time = asyncio.get_event_loop().time()
-        
-    except Exception as e:
-        logger.error(f"Test failed with error: {str(e)}", tag="TEST")
-        results.errors.append(e)
-    
-    # Log summary
-    results.log_summary(logger)
-    
-    return results
-
-# ======== RUN ALL TESTS ========
-async def run_all_tests():
-    """Run all test scenarios"""
-    logger = AsyncLogger(verbose=True)
-    logger.info("=== STARTING BROWSER HUB TESTS ===", tag="MAIN")
-    
-    try:
-        # Run each test scenario
-        await test_default_configuration()
-        # await test_custom_configuration()
-        # await test_preinitalized_browser_hub()
-        # await test_parallel_pipelines()
-        # await test_connection_string()
-        
-    except Exception as e:
-        logger.error(f"Test suite failed: {str(e)}", tag="MAIN")
-    finally:
-        # Clean up all browser hubs
-        logger.info("Shutting down all browser hubs...", tag="MAIN")
-        await BrowserHub.shutdown_all()
-        logger.success("All tests completed", tag="MAIN")
-
-if __name__ == "__main__":
-    asyncio.run(run_all_tests())

tests/pipeline/test_batch_crawl.py

@@ -1,163 +0,0 @@
-"""Test the Crawler class for batch crawling capabilities."""
-
-import asyncio
-import pytest
-from typing import List, Dict, Any, Optional, Tuple
-
-from crawl4ai import Crawler
-from crawl4ai import BrowserConfig, CrawlerRunConfig
-from crawl4ai.async_logger import AsyncLogger
-from crawl4ai.models import CrawlResult, CrawlResultContainer
-from crawl4ai.browser import BrowserHub
-from crawl4ai.cache_context import CacheMode
-
-# Test URLs for crawling
-SAFE_URLS = [
-   "https://example.com",
-   "https://httpbin.org/html",
-   "https://httpbin.org/headers",
-   "https://httpbin.org/ip",
-   "https://httpbin.org/user-agent",
-   "https://httpstat.us/200",
-   "https://jsonplaceholder.typicode.com/posts/1",
-   "https://jsonplaceholder.typicode.com/comments/1",
-   "https://iana.org",
-   "https://www.python.org"
-]
-
-# Simple test for batch crawling
-@pytest.mark.asyncio
-async def test_batch_crawl_simple():
-    """Test simple batch crawling with multiple URLs."""
-    # Use a few test URLs
-    urls = SAFE_URLS[:3]
-    
-    # Custom crawler config
-    crawler_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-        wait_until="domcontentloaded"
-    )
-    
-    # Crawl multiple URLs using batch crawl
-    results = await Crawler.crawl(
-        urls,
-        crawler_config=crawler_config
-    )
-    
-    # Verify the results
-    assert isinstance(results, dict)
-    assert len(results) == len(urls)
-    
-    for url in urls:
-        assert url in results
-        assert results[url].success
-        assert results[url].html is not None
-
-# Test parallel batch crawling
-@pytest.mark.asyncio
-async def test_parallel_batch_crawl():
-    """Test parallel batch crawling with multiple URLs."""
-    # Use several URLs for parallel crawling
-    urls = SAFE_URLS[:5]
-    
-    # Basic crawler config
-    crawler_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-        wait_until="domcontentloaded"
-    )
-    
-    # Crawl in parallel
-    start_time = asyncio.get_event_loop().time()
-    results = await Crawler.parallel_crawl(
-        urls,
-        crawler_config=crawler_config
-    )
-    end_time = asyncio.get_event_loop().time()
-    
-    # Verify results
-    assert len(results) == len(urls)
-    successful = sum(1 for r in results.values() if r.success)
-    
-    print(f"Parallel crawl of {len(urls)} URLs completed in {end_time - start_time:.2f}s")
-    print(f"Success rate: {successful}/{len(urls)}")
-    
-    # At least 80% should succeed
-    assert successful / len(urls) >= 0.8
-
-# Test batch crawling with different configurations
-@pytest.mark.asyncio
-async def test_batch_crawl_mixed_configs():
-    """Test batch crawling with different configurations for different URLs."""
-    # Create URL batches with different configurations
-    batch1 = (SAFE_URLS[:2], CrawlerRunConfig(wait_until="domcontentloaded", screenshot=False))
-    batch2 = (SAFE_URLS[2:4], CrawlerRunConfig(wait_until="networkidle", screenshot=True))
-    
-    # Crawl with mixed configurations
-    start_time = asyncio.get_event_loop().time()
-    results = await Crawler.parallel_crawl([batch1, batch2])
-    end_time = asyncio.get_event_loop().time()
-    
-    # Extract all URLs
-    all_urls = batch1[0] + batch2[0]
-    
-    # Verify results
-    assert len(results) == len(all_urls)
-    
-    # Check that screenshots are present only for batch2
-    for url in batch1[0]:
-        assert results[url].screenshot is None
-    
-    for url in batch2[0]:
-        assert results[url].screenshot is not None
-    
-    print(f"Mixed-config parallel crawl of {len(all_urls)} URLs completed in {end_time - start_time:.2f}s")
-
-# Test shared browser hub
-@pytest.mark.asyncio
-async def test_batch_crawl_shared_hub():
-    """Test batch crawling with a shared browser hub."""
-    # Create and initialize a browser hub
-    browser_config = BrowserConfig(
-        browser_type="chromium",
-        headless=True
-    )
-    
-    browser_hub = await BrowserHub.get_browser_manager(
-        config=browser_config,
-        max_browsers_per_config=3,
-        max_pages_per_browser=4,
-        initial_pool_size=1
-    )
-    
-    try:
-        # Use the hub for parallel crawling
-        urls = SAFE_URLS[:3]
-        
-        start_time = asyncio.get_event_loop().time()
-        results = await Crawler.parallel_crawl(
-            urls,
-            browser_hub=browser_hub,
-            crawler_config=CrawlerRunConfig(
-                cache_mode=CacheMode.BYPASS,
-                wait_until="domcontentloaded"
-            )
-        )
-        end_time = asyncio.get_event_loop().time()
-        
-        # Verify results
-        assert len(results) == len(urls)
-        successful = sum(1 for r in results.values() if r.success)
-        
-        print(f"Shared hub parallel crawl of {len(urls)} URLs completed in {end_time - start_time:.2f}s")
-        print(f"Success rate: {successful}/{len(urls)}")
-        
-        # Get browser hub statistics
-        hub_stats = await browser_hub.get_pool_status()
-        print(f"Browser hub stats: {hub_stats}")
-        
-        # At least 80% should succeed
-        assert successful / len(urls) >= 0.8
-        
-    finally:
-        # Clean up the browser hub
-        await browser_hub.close()

tests/pipeline/test_crawler.py

@@ -1,447 +0,0 @@
-# test_crawler.py
-import asyncio
-import warnings
-import pytest
-import pytest_asyncio
-from typing import Optional, Tuple
-
-# Define test fixtures
-@pytest_asyncio.fixture
-async def clean_browser_hub():
-    """Fixture to ensure clean browser hub state between tests."""
-    # Yield control to the test
-    yield
-    
-    # After test, cleanup all browser hubs
-    from crawl4ai.browser import BrowserHub
-    try:
-        await BrowserHub.shutdown_all()
-    except Exception as e:
-        print(f"Error during browser cleanup: {e}")
-
-from crawl4ai import Crawler
-from crawl4ai import BrowserConfig, CrawlerRunConfig
-from crawl4ai.async_logger import AsyncLogger
-from crawl4ai.models import CrawlResultContainer
-from crawl4ai.browser import BrowserHub
-from crawl4ai.cache_context import CacheMode
-
-import warnings
-from pydantic import PydanticDeprecatedSince20
-
-
-
-# Test URLs for crawling
-SAFE_URLS = [
-    "https://example.com",
-    "https://httpbin.org/html",
-    "https://httpbin.org/headers",
-    "https://httpbin.org/ip",
-    "https://httpbin.org/user-agent",
-    "https://httpstat.us/200",
-    "https://jsonplaceholder.typicode.com/posts/1",
-    "https://jsonplaceholder.typicode.com/comments/1",
-    "https://iana.org",
-    "https://www.python.org",
-]
-
-
-class TestCrawlerBasic:
-    """Basic tests for the Crawler utility class"""
-
-    @pytest.mark.asyncio
-    async def test_simple_crawl_single_url(self, clean_browser_hub):
-        """Test crawling a single URL with default configuration"""
-        with warnings.catch_warnings():
-            warnings.filterwarnings("ignore", category=Warning)
-            # Basic logger
-            logger = AsyncLogger(verbose=True)
-
-            # Basic single URL crawl with default configuration
-            url = "https://example.com"
-            result = await Crawler.crawl(url)
-
-            # Verify the result
-            assert isinstance(result, CrawlResultContainer)
-            assert result.success
-            assert result.url == url
-            assert result.html is not None
-            assert len(result.html) > 0
-
-    @pytest.mark.asyncio
-    async def test_crawl_with_custom_config(self, clean_browser_hub):
-        """Test crawling with custom browser and crawler configuration"""
-        # Custom browser config
-        browser_config = BrowserConfig(
-            browser_type="chromium",
-            headless=True,
-            viewport_width=1280,
-            viewport_height=800,
-        )
-
-        # Custom crawler config
-        crawler_config = CrawlerRunConfig(
-            cache_mode=CacheMode.BYPASS, wait_until="networkidle", screenshot=True
-        )
-
-        # Crawl with custom configuration
-        url = "https://httpbin.org/html"
-        result = await Crawler.crawl(
-            url, browser_config=browser_config, crawler_config=crawler_config
-        )
-
-        # Verify the result
-        assert result.success
-        assert result.url == url
-        assert result.screenshot is not None
-
-    @pytest.mark.asyncio
-    async def test_crawl_multiple_urls_sequential(self, clean_browser_hub):
-        """Test crawling multiple URLs sequentially"""
-        # Use a few test URLs
-        urls = SAFE_URLS[:3]
-
-        # Custom crawler config
-        crawler_config = CrawlerRunConfig(
-            cache_mode=CacheMode.BYPASS, wait_until="domcontentloaded"
-        )
-
-        # Crawl multiple URLs sequentially
-        results = await Crawler.crawl(urls, crawler_config=crawler_config)
-
-        # Verify the results
-        assert isinstance(results, dict)
-        assert len(results) == len(urls)
-
-        for url in urls:
-            assert url in results
-            assert results[url].success
-            assert results[url].html is not None
-
-    @pytest.mark.asyncio
-    async def test_crawl_with_error_handling(self, clean_browser_hub):
-        """Test error handling during crawling"""
-        # Include a valid URL and a non-existent URL
-        urls = ["https://example.com", "https://non-existent-domain-123456789.com"]
-
-        # Crawl with retries
-        results = await Crawler.crawl(urls, max_retries=2, retry_delay=1.0)
-
-        # Verify results for both URLs
-        assert len(results) == 2
-
-        # Valid URL should succeed
-        assert results[urls[0]].success
-
-        # Invalid URL should fail but be in results
-        assert urls[1] in results
-        assert not results[urls[1]].success
-        assert results[urls[1]].error_message is not None
-
-
-class TestCrawlerParallel:
-    """Tests for the parallel crawling capabilities of Crawler"""
-
-    @pytest.mark.asyncio
-    async def test_parallel_crawl_simple(self, clean_browser_hub):
-        """Test basic parallel crawling with same configuration"""
-        # Use several URLs for parallel crawling
-        urls = SAFE_URLS[:5]
-
-        # Basic crawler config
-        crawler_config = CrawlerRunConfig(
-            cache_mode=CacheMode.BYPASS, wait_until="domcontentloaded"
-        )
-
-        # Crawl in parallel with default concurrency
-        start_time = asyncio.get_event_loop().time()
-        results = await Crawler.parallel_crawl(urls, crawler_config=crawler_config)
-        end_time = asyncio.get_event_loop().time()
-
-        # Verify results
-        assert len(results) == len(urls)
-        successful = sum(1 for r in results.values() if r.success)
-
-        print(
-            f"Parallel crawl of {len(urls)} URLs completed in {end_time - start_time:.2f}s"
-        )
-        print(f"Success rate: {successful}/{len(urls)}")
-
-        # At least 80% should succeed
-        assert successful / len(urls) >= 0.8
-
-    @pytest.mark.asyncio
-    async def test_parallel_crawl_with_concurrency_limit(self, clean_browser_hub):
-        """Test parallel crawling with concurrency limit"""
-        # Use more URLs to test concurrency control
-        urls = SAFE_URLS[:8]
-
-        # Custom crawler config
-        crawler_config = CrawlerRunConfig(
-            cache_mode=CacheMode.BYPASS, wait_until="domcontentloaded"
-        )
-
-        # Limited concurrency
-        concurrency = 2
-
-        # Time the crawl
-        start_time = asyncio.get_event_loop().time()
-        results = await Crawler.parallel_crawl(
-            urls, crawler_config=crawler_config, concurrency=concurrency
-        )
-        end_time = asyncio.get_event_loop().time()
-
-        # Verify results
-        assert len(results) == len(urls)
-        successful = sum(1 for r in results.values() if r.success)
-
-        print(
-            f"Parallel crawl with concurrency={concurrency} of {len(urls)} URLs completed in {end_time - start_time:.2f}s"
-        )
-        print(f"Success rate: {successful}/{len(urls)}")
-
-        # At least 80% should succeed
-        assert successful / len(urls) >= 0.8
-
-    @pytest.mark.asyncio
-    async def test_parallel_crawl_with_different_configs(self, clean_browser_hub):
-        """Test parallel crawling with different configurations for different URLs"""
-        # Create URL batches with different configurations
-        batch1 = (
-            SAFE_URLS[:2],
-            CrawlerRunConfig(wait_until="domcontentloaded", screenshot=False),
-        )
-        batch2 = (
-            SAFE_URLS[2:4],
-            CrawlerRunConfig(wait_until="networkidle", screenshot=True),
-        )
-        batch3 = (
-            SAFE_URLS[4:6],
-            CrawlerRunConfig(wait_until="load", scan_full_page=True),
-        )
-
-        # Crawl with mixed configurations
-        start_time = asyncio.get_event_loop().time()
-        results = await Crawler.parallel_crawl([batch1, batch2, batch3])
-        end_time = asyncio.get_event_loop().time()
-
-        # Extract all URLs
-        all_urls = batch1[0] + batch2[0] + batch3[0]
-
-        # Verify results
-        assert len(results) == len(all_urls)
-
-        # Check that screenshots are present only for batch2
-        for url in batch1[0]:
-            assert not results[url].screenshot
-
-        for url in batch2[0]:
-            assert results[url].screenshot
-
-        print(
-            f"Mixed-config parallel crawl of {len(all_urls)} URLs completed in {end_time - start_time:.2f}s"
-        )
-
-    @pytest.mark.asyncio
-    async def test_parallel_crawl_with_shared_browser_hub(self, clean_browser_hub):
-        """Test parallel crawling with a shared browser hub"""
-        # Create and initialize a browser hub
-        browser_config = BrowserConfig(browser_type="chromium", headless=True)
-
-        browser_hub = await BrowserHub.get_browser_manager(
-            config=browser_config,
-            max_browsers_per_config=3,
-            max_pages_per_browser=4,
-            initial_pool_size=1,
-        )
-
-        try:
-            # Use the hub for parallel crawling
-            urls = SAFE_URLS[:6]
-
-            start_time = asyncio.get_event_loop().time()
-            results = await Crawler.parallel_crawl(
-                urls,
-                browser_hub=browser_hub,
-                crawler_config=CrawlerRunConfig(
-                    cache_mode=CacheMode.BYPASS, wait_until="domcontentloaded"
-                ),
-            )
-            end_time = asyncio.get_event_loop().time()
-
-            # Verify results
-            # assert (len(results), len(urls))
-            assert len(results) == len(urls)
-            successful = sum(1 for r in results.values() if r.success)
-
-            print(
-                f"Shared hub parallel crawl of {len(urls)} URLs completed in {end_time - start_time:.2f}s"
-            )
-            print(f"Success rate: {successful}/{len(urls)}")
-
-            # Get browser hub statistics
-            hub_stats = await browser_hub.get_pool_status()
-            print(f"Browser hub stats: {hub_stats}")
-
-            # At least 80% should succeed
-            # assert (successful / len(urls), 0.8)
-            assert successful / len(urls) >= 0.8
-
-        finally:
-            # Clean up the browser hub
-            await browser_hub.close()
-
-
-class TestCrawlerAdvanced:
-    """Advanced tests for the Crawler utility class"""
-
-    @pytest.mark.asyncio
-    async def test_crawl_with_customized_batch_config(self, clean_browser_hub):
-        """Test crawling with fully customized batch configuration"""
-        # Create URL batches with different browser and crawler configurations
-        browser_config1 = BrowserConfig(browser_type="chromium", headless=True)
-        browser_config2 = BrowserConfig(
-            browser_type="chromium", headless=False, viewport_width=1920
-        )
-
-        crawler_config1 = CrawlerRunConfig(wait_until="domcontentloaded")
-        crawler_config2 = CrawlerRunConfig(wait_until="networkidle", screenshot=True)
-
-        batch1 = (SAFE_URLS[:2], browser_config1, crawler_config1)
-        batch2 = (SAFE_URLS[2:4], browser_config2, crawler_config2)
-
-        # Crawl with mixed configurations
-        results = await Crawler.parallel_crawl([batch1, batch2])
-
-        # Extract all URLs
-        all_urls = batch1[0] + batch2[0]
-
-        # Verify results
-        # assert (len(results), len(all_urls))
-        assert len(results) == len(all_urls)
-
-        # Verify batch-specific processing
-        for url in batch1[0]:
-            assert results[url].screenshot is None  # No screenshots for batch1
-
-        for url in batch2[0]:
-            assert results[url].screenshot is not None  # Should have screenshots for batch2
-
-    @pytest.mark.asyncio
-    async def test_crawl_with_progress_callback(self, clean_browser_hub):
-        """Test crawling with progress callback"""
-        # Use several URLs
-        urls = SAFE_URLS[:5]
-
-        # Track progress
-        progress_data = {"started": 0, "completed": 0, "failed": 0, "updates": []}
-
-        # Progress callback
-        async def on_progress(
-            status: str, url: str, result: Optional[CrawlResultContainer] = None
-        ):
-            if status == "started":
-                progress_data["started"] += 1
-            elif status == "completed":
-                progress_data["completed"] += 1
-                if not result.success:
-                    progress_data["failed"] += 1
-
-            progress_data["updates"].append((status, url))
-            print(f"Progress: {status} - {url}")
-
-        # Crawl with progress tracking
-        results = await Crawler.parallel_crawl(
-            urls,
-            crawler_config=CrawlerRunConfig(wait_until="domcontentloaded"),
-            progress_callback=on_progress,
-        )
-
-        # Verify progress tracking
-        assert progress_data["started"] == len(urls)
-        assert progress_data["completed"] == len(urls)
-        assert len(progress_data["updates"]) == len(urls) * 2  # start + complete events
-
-    @pytest.mark.asyncio
-    async def test_crawl_with_dynamic_retry_strategy(self, clean_browser_hub):
-        """Test crawling with a dynamic retry strategy"""
-        # Include URLs that might fail
-        urls = [
-            "https://example.com",
-            "https://httpstat.us/500",
-            "https://httpstat.us/404",
-        ]
-
-        # Custom retry strategy
-        async def retry_strategy(
-            url: str, attempt: int, error: Exception
-        ) -> Tuple[bool, float]:
-            # Only retry 500 errors, not 404s
-            if "500" in url:
-                return True, 1.0  # Retry with 1 second delay
-            return False, 0.0  # Don't retry other errors
-
-        # Crawl with custom retry strategy
-        results = await Crawler.parallel_crawl(
-            urls,
-            crawler_config=CrawlerRunConfig(wait_until="domcontentloaded"),
-            retry_strategy=retry_strategy,
-            max_retries=3,
-        )
-
-        # Verify results
-        assert len(results) == len(urls)
-
-        # Example.com should succeed
-        assert results[urls[0]].success
-
-        # httpstat.us pages return content even for error status codes
-        # so our crawler marks them as successful since it got HTML content
-        # Verify that we got the expected status code
-        assert results[urls[1]].status_code == 500
-        
-        # 404 should have the correct status code
-        assert results[urls[2]].status_code == 404
-
-    @pytest.mark.asyncio
-    async def test_crawl_with_very_large_batch(self, clean_browser_hub):
-        """Test crawling with a very large batch of URLs"""
-        # Create a batch by repeating our safe URLs
-        # Note: In a real test, we'd use more URLs, but for simplicity we'll use a smaller set
-        large_batch = list(dict.fromkeys(SAFE_URLS[:5] * 2))  # ~10 unique URLs
-
-        # Set a reasonable concurrency limit
-        concurrency = 10
-
-        # Time the crawl
-        start_time = asyncio.get_event_loop().time()
-        results = await Crawler.parallel_crawl(
-            large_batch,
-            crawler_config=CrawlerRunConfig(
-                wait_until="domcontentloaded",
-                page_timeout=10000,  # Shorter timeout for large batch
-            ),
-            concurrency=concurrency,
-        )
-        end_time = asyncio.get_event_loop().time()
-
-        # Verify results
-        # assert (len(results), len(large_batch))
-        assert len(results) == len(large_batch)
-        successful = sum(1 for r in results.values() if r.success)
-
-        print(
-            f"Large batch crawl of {len(large_batch)} URLs completed in {end_time - start_time:.2f}s"
-        )
-        print(f"Success rate: {successful}/{len(large_batch)}")
-        print(
-            f"Average time per URL: {(end_time - start_time) / len(large_batch):.2f}s"
-        )
-
-        # At least 80% should succeed (from our unique URLs)
-        assert successful / len(results) >= 0.8
-
-
-if __name__ == "__main__":
-    # Use pytest for async tests
-    pytest.main(["-xvs", __file__])

tests/pipeline/test_pipeline.py

@@ -1,109 +0,0 @@
-import asyncio
-from crawl4ai import (
-    BrowserConfig,
-    CrawlerRunConfig,
-    CacheMode,
-    DefaultMarkdownGenerator,
-    PruningContentFilter
-)
-from pipeline import Pipeline
-
-async def main():
-    # Create configuration objects
-    browser_config = BrowserConfig(headless=True, verbose=True)
-    crawler_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-        markdown_generator=DefaultMarkdownGenerator(
-            content_filter=PruningContentFilter(
-                threshold=0.48, 
-                threshold_type="fixed", 
-                min_word_threshold=0
-            )
-        ),
-    )
-    
-    # Create and use pipeline with context manager
-    async with Pipeline(browser_config=browser_config) as pipeline:
-        result = await pipeline.crawl(
-            url="https://www.example.com", 
-            config=crawler_config
-        )
-        
-        # Print the result
-        print(f"URL: {result.url}")
-        print(f"Success: {result.success}")
-        
-        if result.success:
-            print("\nMarkdown excerpt:")
-            print(result.markdown.raw_markdown[:500] + "...")
-        else:
-            print(f"Error: {result.error_message}")
-
-if __name__ == "__main__":
-    asyncio.run(main())
-
-
-class CrawlTarget:
-    def __init__(self, urls, config=None):
-        self.urls = urls
-        self.config = config
-
-    def __repr__(self):
-        return f"CrawlTarget(urls={self.urls}, config={self.config})"
-    
-
-
-
-# async def main():
-#     # Create configuration objects
-#     browser_config = BrowserConfig(headless=True, verbose=True)
-    
-#     # Define different configurations
-#     config1 = CrawlerRunConfig(
-#         cache_mode=CacheMode.BYPASS,
-#         markdown_generator=DefaultMarkdownGenerator(
-#             content_filter=PruningContentFilter(threshold=0.48)
-#         ),
-#     )
-    
-#     config2 = CrawlerRunConfig(
-#         cache_mode=CacheMode.ENABLED,
-#         screenshot=True,
-#         pdf=True
-#     )
-    
-#     # Create crawl targets
-#     targets = [
-#         CrawlTarget(
-#             urls=["https://www.example.com", "https://www.wikipedia.org"],
-#             config=config1
-#         ),
-#         CrawlTarget(
-#             urls="https://news.ycombinator.com",  
-#             config=config2
-#         ),
-#         CrawlTarget(
-#             urls=["https://github.com", "https://stackoverflow.com", "https://python.org"],
-#             config=None
-#         )
-#     ]
-    
-#     # Create and use pipeline with context manager
-#     async with Pipeline(browser_config=browser_config) as pipeline:
-#         all_results = await pipeline.crawl_batch(targets)
-        
-#         for target_key, results in all_results.items():
-#             print(f"\n===== Results for {target_key} =====")
-#             print(f"Number of URLs crawled: {len(results)}")
-            
-#             for i, result in enumerate(results):
-#                 print(f"\nURL {i+1}: {result.url}")
-#                 print(f"Success: {result.success}")
-                
-#                 if result.success:
-#                     print(f"Content length: {len(result.markdown.raw_markdown)} chars")
-#                 else:
-#                     print(f"Error: {result.error_message}")
-
-# if __name__ == "__main__":
-#     asyncio.run(main())