Add all 5 deployments solution for testing

Adds new target_elements parameter to CrawlerRunConfig that allows more flexible content selection than css_selector. This enables focusing markdown generation and data extraction on specific elements while still processing the entire page for links and media.

Key changes:
- Added target_elements list parameter to CrawlerRunConfig
- Modified WebScrapingStrategy and LXMLWebScrapingStrategy to handle target_elements
- Updated documentation with examples and comparison between css_selector and target_elements
- Fixed table extraction in content_scraping_strategy.py

BREAKING CHANGE: Table extraction logic has been modified to better handle thead/tbody structures

crawl4ai/__init__.py

@@ -23,6 +23,7 @@ from .extraction_strategy import (
     CosineStrategy,
     JsonCssExtractionStrategy,
     JsonXPathExtractionStrategy,
+    JsonLxmlExtractionStrategy
 )
 from .chunking_strategy import ChunkingStrategy, RegexChunking
 from .markdown_generation_strategy import DefaultMarkdownGenerator
@@ -103,6 +104,7 @@ __all__ = [
     "CosineStrategy",
     "JsonCssExtractionStrategy",
     "JsonXPathExtractionStrategy",
+    "JsonLxmlExtractionStrategy",
     "ChunkingStrategy",
     "RegexChunking",
     "DefaultMarkdownGenerator",

crawl4ai/__version__.py

@@ -1,2 +1,2 @@
 # crawl4ai/_version.py
-__version__ = "0.5.0.post6"
+__version__ = "0.5.0.post4"

crawl4ai/async_configs.py

@@ -11,7 +11,7 @@ from .config import (
 )
 
 from .user_agent_generator import UAGen, ValidUAGenerator  # , OnlineUAGenerator
-from .extraction_strategy import ExtractionStrategy
+from .extraction_strategy import ExtractionStrategy, LLMExtractionStrategy
 from .chunking_strategy import ChunkingStrategy, RegexChunking
 
 from .markdown_generation_strategy import MarkdownGenerationStrategy
@@ -26,6 +26,8 @@ import inspect
 from typing import Any, Dict, Optional
 from enum import Enum
 
+from .proxy_strategy import ProxyConfig
+
 
 def to_serializable_dict(obj: Any, ignore_default_value : bool = False) -> Dict:
     """
@@ -180,7 +182,7 @@ class BrowserConfig:
                               is "chromium". Default: "chromium".
         proxy (Optional[str]): Proxy server URL (e.g., "http://username:password@proxy:port"). If None, no proxy is used.
                              Default: None.
-        proxy_config (dict or None): Detailed proxy configuration, e.g. {"server": "...", "username": "..."}.
+        proxy_config (ProxyConfig or dict or None): Detailed proxy configuration, e.g. {"server": "...", "username": "..."}.
                                      If None, no additional proxy config. Default: None.
         viewport_width (int): Default viewport width for pages. Default: 1080.
         viewport_height (int): Default viewport height for pages. Default: 600.
@@ -225,7 +227,7 @@ class BrowserConfig:
         chrome_channel: str = "chromium",
         channel: str = "chromium",
         proxy: str = None,
-        proxy_config: dict = None,
+        proxy_config: Union[ProxyConfig, dict, None] = None,
         viewport_width: int = 1080,
         viewport_height: int = 600,
         viewport: dict = None,
@@ -315,7 +317,7 @@ class BrowserConfig:
             chrome_channel=kwargs.get("chrome_channel", "chromium"),
             channel=kwargs.get("channel", "chromium"),
             proxy=kwargs.get("proxy"),
-            proxy_config=kwargs.get("proxy_config"),
+            proxy_config=kwargs.get("proxy_config", None),
             viewport_width=kwargs.get("viewport_width", 1080),
             viewport_height=kwargs.get("viewport_height", 600),
             accept_downloads=kwargs.get("accept_downloads", False),
@@ -499,6 +501,15 @@ class CrawlerRunConfig():
                           Default: False.
         css_selector (str or None): CSS selector to extract a specific portion of the page.
                                     Default: None.
+        
+        target_elements (list of str or None): List of CSS selectors for specific elements for Markdown generation 
+                                                and structured data extraction. When you set this, only the contents 
+                                                of these elements are processed for extraction and Markdown generation. 
+                                                If you do not set any value, the entire page is processed. 
+                                                The difference between this and css_selector is that this will shrink 
+                                                the initial raw HTML to the selected element, while this will only affect 
+                                                the extraction and Markdown generation.
+                                    Default: None
         excluded_tags (list of str or None): List of HTML tags to exclude from processing.
                                              Default: None.
         excluded_selector (str or None): CSS selector to exclude from processing.
@@ -515,7 +526,7 @@ class CrawlerRunConfig():
                            Default: "lxml".
         scraping_strategy (ContentScrapingStrategy): Scraping strategy to use.
                            Default: WebScrapingStrategy.
-        proxy_config (dict or None): Detailed proxy configuration, e.g. {"server": "...", "username": "..."}.
+        proxy_config (ProxyConfig or dict or None): Detailed proxy configuration, e.g. {"server": "...", "username": "..."}.
                                      If None, no additional proxy config. Default: None.
 
         # SSL Parameters
@@ -595,6 +606,8 @@ class CrawlerRunConfig():
                                      Default: IMAGE_SCORE_THRESHOLD (e.g., 3).
         exclude_external_images (bool): If True, exclude all external images from processing.
                                          Default: False.
+        table_score_threshold (int): Minimum score threshold for processing a table.
+                                     Default: 7.
 
         # Link and Domain Handling Parameters
         exclude_social_media_domains (list of str): List of domains to exclude for social media links.
@@ -648,6 +661,7 @@ class CrawlerRunConfig():
         markdown_generator: MarkdownGenerationStrategy = None,
         only_text: bool = False,
         css_selector: str = None,
+        target_elements: List[str] = None,
         excluded_tags: list = None,
         excluded_selector: str = None,
         keep_data_attributes: bool = False,
@@ -656,7 +670,7 @@ class CrawlerRunConfig():
         prettiify: bool = False,
         parser_type: str = "lxml",
         scraping_strategy: ContentScrapingStrategy = None,
-        proxy_config: dict = None,
+        proxy_config: Union[ProxyConfig, dict, None] = None,
         proxy_rotation_strategy: Optional[ProxyRotationStrategy] = None,
         # SSL Parameters
         fetch_ssl_certificate: bool = False,
@@ -696,6 +710,7 @@ class CrawlerRunConfig():
         pdf: bool = False,
         image_description_min_word_threshold: int = IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD,
         image_score_threshold: int = IMAGE_SCORE_THRESHOLD,
+        table_score_threshold: int = 7,
         exclude_external_images: bool = False,
         # Link and Domain Handling Parameters
         exclude_social_media_domains: list = None,
@@ -727,6 +742,7 @@ class CrawlerRunConfig():
         self.markdown_generator = markdown_generator
         self.only_text = only_text
         self.css_selector = css_selector
+        self.target_elements = target_elements or []
         self.excluded_tags = excluded_tags or []
         self.excluded_selector = excluded_selector or ""
         self.keep_data_attributes = keep_data_attributes
@@ -781,6 +797,7 @@ class CrawlerRunConfig():
         self.image_description_min_word_threshold = image_description_min_word_threshold
         self.image_score_threshold = image_score_threshold
         self.exclude_external_images = exclude_external_images
+        self.table_score_threshold = table_score_threshold
 
         # Link and Domain Handling Parameters
         self.exclude_social_media_domains = (
@@ -856,6 +873,7 @@ class CrawlerRunConfig():
             markdown_generator=kwargs.get("markdown_generator"),
             only_text=kwargs.get("only_text", False),
             css_selector=kwargs.get("css_selector"),
+            target_elements=kwargs.get("target_elements", []),
             excluded_tags=kwargs.get("excluded_tags", []),
             excluded_selector=kwargs.get("excluded_selector", ""),
             keep_data_attributes=kwargs.get("keep_data_attributes", False),
@@ -911,6 +929,7 @@ class CrawlerRunConfig():
             image_score_threshold=kwargs.get(
                 "image_score_threshold", IMAGE_SCORE_THRESHOLD
             ),
+            table_score_threshold=kwargs.get("table_score_threshold", 7),
             exclude_external_images=kwargs.get("exclude_external_images", False),
             # Link and Domain Handling Parameters
             exclude_social_media_domains=kwargs.get(
@@ -956,6 +975,7 @@ class CrawlerRunConfig():
             "markdown_generator": self.markdown_generator,
             "only_text": self.only_text,
             "css_selector": self.css_selector,
+            "target_elements": self.target_elements,
             "excluded_tags": self.excluded_tags,
             "excluded_selector": self.excluded_selector,
             "keep_data_attributes": self.keep_data_attributes,
@@ -999,6 +1019,7 @@ class CrawlerRunConfig():
             "pdf": self.pdf,
             "image_description_min_word_threshold": self.image_description_min_word_threshold,
             "image_score_threshold": self.image_score_threshold,
+            "table_score_threshold": self.table_score_threshold,
             "exclude_external_images": self.exclude_external_images,
             "exclude_social_media_domains": self.exclude_social_media_domains,
             "exclude_external_links": self.exclude_external_links,
@@ -1091,3 +1112,5 @@ class LLMConfig:
         config_dict = self.to_dict()
         config_dict.update(kwargs)
         return LLMConfig.from_kwargs(config_dict)
+
+

crawl4ai/async_crawler_strategy.py

@@ -767,6 +767,7 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
             # Handle wait_for condition
             # Todo: Decide how to handle this
             if not config.wait_for and config.css_selector and False:
+            # if not config.wait_for and config.css_selector:
                 config.wait_for = f"css:{config.css_selector}"
 
             if config.wait_for:
@@ -806,8 +807,28 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
             if config.remove_overlay_elements:
                 await self.remove_overlay_elements(page)
 
-            # Get final HTML content
-            html = await page.content()
+            if config.css_selector:
+                try:
+                    # Handle comma-separated selectors by splitting them
+                    selectors = [s.strip() for s in config.css_selector.split(',')]
+                    html_parts = []
+                    
+                    for selector in selectors:
+                        try:
+                            content = await page.evaluate(f"document.querySelector('{selector}')?.outerHTML || ''")
+                            html_parts.append(content)
+                        except Error as e:
+                            print(f"Warning: Could not get content for selector '{selector}': {str(e)}")
+                    
+                    # Wrap in a div to create a valid HTML structure
+                    html = f"<div class='crawl4ai-result'>\n" + "\n".join(html_parts) + "\n</div>"                    
+                except Error as e:
+                    raise RuntimeError(f"Failed to extract HTML content: {str(e)}")
+            else:
+                html = await page.content()
+            
+            # # Get final HTML content
+            # html = await page.content()
             await self.execute_hook(
                 "before_return_html", page=page, html=html, context=context, config=config
             )

crawl4ai/async_webcrawler.py

@@ -514,7 +514,8 @@ class AsyncWebCrawler:
                 scraping_strategy.logger = self.logger
 
             # Process HTML content
-            params = {k: v for k, v in config.to_dict().items() if k not in ["url"]}
+            params = config.__dict__.copy()
+            params.pop("url", None)            
             # add keys from kwargs to params that doesn't exist in params
             params.update({k: v for k, v in kwargs.items() if k not in params.keys()})
 

crawl4ai/browser_manager.py

@@ -434,8 +434,9 @@ class BrowserManager:
 
             self.playwright = await async_playwright().start()
 
-        if self.config.use_managed_browser:
-            cdp_url = await self.managed_browser.start()
+        if self.config.cdp_url or self.config.use_managed_browser:
+            self.config.use_managed_browser = True
+            cdp_url = await self.managed_browser.start() if not self.config.cdp_url else self.config.cdp_url
             self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url)
             contexts = self.browser.contexts
             if contexts:
@@ -530,9 +531,9 @@ class BrowserManager:
                 ProxySettings(server=self.config.proxy)
                 if self.config.proxy
                 else ProxySettings(
-                    server=self.config.proxy_config.get("server"),
-                    username=self.config.proxy_config.get("username"),
-                    password=self.config.proxy_config.get("password"),
+                    server=self.config.proxy_config.server,
+                    username=self.config.proxy_config.username,
+                    password=self.config.proxy_config.password,
                 )
             )
             browser_args["proxy"] = proxy_settings
@@ -790,7 +791,10 @@ class BrowserManager:
         # If using a managed browser, just grab the shared default_context
         if self.config.use_managed_browser:
             context = self.default_context
-            page = await context.new_page()
+            pages = context.pages
+            page = next((p for p in pages if p.url == crawlerRunConfig.url), None)
+            if not page:
+                page = await context.new_page()
         else:
             # Otherwise, check if we have an existing context for this config
             config_signature = self._make_config_signature(crawlerRunConfig)
@@ -840,6 +844,9 @@ class BrowserManager:
 
     async def close(self):
         """Close all browser resources and clean up."""
+        if self.config.cdp_url:
+            return
+        
         if self.config.sleep_on_close:
             await asyncio.sleep(0.5)
 

crawl4ai/browser_profiler.py

@@ -342,7 +342,11 @@ class BrowserProfiler:
         
         # Check if path exists and is a valid profile
         if not os.path.isdir(profile_path):
-            return None
+            # Chrck if profile_name itself is full path
+            if os.path.isabs(profile_name):
+                profile_path = profile_name
+            else:
+                return None
         
         # Look for profile indicators
         is_profile = (
@@ -541,4 +545,225 @@ class BrowserProfiler:
                 break
                 
             else:
-                self.logger.error(f"Invalid choice. Please enter a number between 1 and {exit_option}.", tag="MENU")
+                self.logger.error(f"Invalid choice. Please enter a number between 1 and {exit_option}.", tag="MENU")
+
+
+    async def launch_standalone_browser(self, 
+                                  browser_type: str = "chromium",
+                                  user_data_dir: Optional[str] = None,
+                                  debugging_port: int = 9222,
+                                  headless: bool = False) -> Optional[str]:
+        """
+        Launch a standalone browser with CDP debugging enabled and keep it running
+        until the user presses 'q'. Returns and displays the CDP URL.
+        
+        Args:
+            browser_type (str): Type of browser to launch ('chromium' or 'firefox')
+            user_data_dir (str, optional): Path to user profile directory
+            debugging_port (int): Port to use for CDP debugging
+            headless (bool): Whether to run in headless mode
+            
+        Returns:
+            str: CDP URL for the browser, or None if launch failed
+            
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            cdp_url = await profiler.launch_standalone_browser(
+                user_data_dir="/path/to/profile",
+                debugging_port=9222
+            )
+            # Use cdp_url to connect to the browser
+            ```
+        """
+        # Use the provided directory if specified, otherwise create a temporary directory
+        if user_data_dir:
+            # Directory is provided directly, ensure it exists
+            profile_path = user_data_dir
+            os.makedirs(profile_path, exist_ok=True)
+        else:
+            # Create a temporary profile directory
+            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+            profile_name = f"temp_{timestamp}_{uuid.uuid4().hex[:6]}"
+            profile_path = os.path.join(self.profiles_dir, profile_name)
+            os.makedirs(profile_path, exist_ok=True)
+        
+        # Print initial information
+        border = f"{Fore.CYAN}{'='*80}{Style.RESET_ALL}"
+        self.logger.info(f"\n{border}", tag="CDP")
+        self.logger.info(f"Launching standalone browser with CDP debugging", tag="CDP")
+        self.logger.info(f"Browser type: {Fore.GREEN}{browser_type}{Style.RESET_ALL}", tag="CDP")
+        self.logger.info(f"Profile path: {Fore.YELLOW}{profile_path}{Style.RESET_ALL}", tag="CDP")
+        self.logger.info(f"Debugging port: {Fore.CYAN}{debugging_port}{Style.RESET_ALL}", tag="CDP")
+        self.logger.info(f"Headless mode: {Fore.CYAN}{headless}{Style.RESET_ALL}", tag="CDP")
+        
+        # Create managed browser instance
+        managed_browser = ManagedBrowser(
+            browser_type=browser_type,
+            user_data_dir=profile_path,
+            headless=headless,
+            logger=self.logger,
+            debugging_port=debugging_port
+        )
+        
+        # Set up signal handlers to ensure cleanup on interrupt
+        original_sigint = signal.getsignal(signal.SIGINT)
+        original_sigterm = signal.getsignal(signal.SIGTERM)
+        
+        # Define cleanup handler for signals
+        async def cleanup_handler(sig, frame):
+            self.logger.warning("\nCleaning up browser process...", tag="CDP")
+            await managed_browser.cleanup()
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            if sig == signal.SIGINT:
+                self.logger.error("Browser terminated by user.", tag="CDP")
+                sys.exit(1)
+                    
+        # Set signal handlers
+        def sigint_handler(sig, frame):
+            asyncio.create_task(cleanup_handler(sig, frame))
+        
+        signal.signal(signal.SIGINT, sigint_handler)
+        signal.signal(signal.SIGTERM, sigint_handler)
+        
+        # Event to signal when user wants to exit
+        user_done_event = asyncio.Event()
+        
+        # Run keyboard input loop in a separate task
+        async def listen_for_quit_command():
+            import termios
+            import tty
+            import select
+            
+            # First output the prompt
+            self.logger.info(f"{Fore.CYAN}Press '{Fore.WHITE}q{Fore.CYAN}' to stop the browser and exit...{Style.RESET_ALL}", tag="CDP")
+            
+            # Save original terminal settings
+            fd = sys.stdin.fileno()
+            old_settings = termios.tcgetattr(fd)
+            
+            try:
+                # Switch to non-canonical mode (no line buffering)
+                tty.setcbreak(fd)
+                
+                while True:
+                    # Check if input is available (non-blocking)
+                    readable, _, _ = select.select([sys.stdin], [], [], 0.5)
+                    if readable:
+                        key = sys.stdin.read(1)
+                        if key.lower() == 'q':
+                            self.logger.info(f"{Fore.GREEN}Closing browser...{Style.RESET_ALL}", tag="CDP")
+                            user_done_event.set()
+                            return
+                    
+                    # Check if the browser process has already exited
+                    if managed_browser.browser_process and managed_browser.browser_process.poll() is not None:
+                        self.logger.info("Browser already closed. Ending input listener.", tag="CDP")
+                        user_done_event.set()
+                        return
+                        
+                    await asyncio.sleep(0.1)
+            
+            finally:
+                # Restore terminal settings 
+                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+                
+        # Function to retrieve and display CDP JSON config
+        async def get_cdp_json(port):
+            import aiohttp
+            cdp_url = f"http://localhost:{port}"
+            json_url = f"{cdp_url}/json/version"
+            
+            try:
+                async with aiohttp.ClientSession() as session:
+                    # Try multiple times in case the browser is still starting up
+                    for _ in range(10):
+                        try:
+                            async with session.get(json_url) as response:
+                                if response.status == 200:
+                                    data = await response.json()
+                                    return cdp_url, data
+                        except Exception:
+                            pass
+                        
+                        await asyncio.sleep(0.5)
+                    
+                    return cdp_url, None
+            except Exception as e:
+                self.logger.error(f"Error fetching CDP JSON: {str(e)}", tag="CDP")
+                return cdp_url, None
+        
+        cdp_url = None
+        config_json = None
+        
+        try:
+            # Start the browser
+            await managed_browser.start()
+            
+            # Check if browser started successfully
+            browser_process = managed_browser.browser_process
+            if not browser_process:
+                self.logger.error("Failed to start browser process.", tag="CDP")
+                return None
+            
+            self.logger.info(f"Browser launched successfully. Retrieving CDP information...", tag="CDP") 
+            
+            # Get CDP URL and JSON config
+            cdp_url, config_json = await get_cdp_json(debugging_port)
+            
+            if cdp_url:
+                self.logger.success(f"CDP URL: {Fore.GREEN}{cdp_url}{Style.RESET_ALL}", tag="CDP")
+                
+                if config_json:
+                    # Display relevant CDP information
+                    self.logger.info(f"Browser: {Fore.CYAN}{config_json.get('Browser', 'Unknown')}{Style.RESET_ALL}", tag="CDP")
+                    self.logger.info(f"Protocol Version: {config_json.get('Protocol-Version', 'Unknown')}", tag="CDP")
+                    if 'webSocketDebuggerUrl' in config_json:
+                        self.logger.info(f"WebSocket URL: {Fore.GREEN}{config_json['webSocketDebuggerUrl']}{Style.RESET_ALL}", tag="CDP")
+                else:
+                    self.logger.warning("Could not retrieve CDP configuration JSON", tag="CDP")
+            else:
+                self.logger.error(f"Failed to get CDP URL on port {debugging_port}", tag="CDP")
+                await managed_browser.cleanup()
+                return None
+            
+            # Start listening for keyboard input
+            listener_task = asyncio.create_task(listen_for_quit_command())
+            
+            # Wait for the user to press 'q' or for the browser process to exit naturally
+            while not user_done_event.is_set() and browser_process.poll() is None:
+                await asyncio.sleep(0.5)
+            
+            # Cancel the listener task if it's still running
+            if not listener_task.done():
+                listener_task.cancel()
+                try:
+                    await listener_task
+                except asyncio.CancelledError:
+                    pass
+            
+            # If the browser is still running and the user pressed 'q', terminate it
+            if browser_process.poll() is None and user_done_event.is_set():
+                self.logger.info("Terminating browser process...", tag="CDP")
+                await managed_browser.cleanup()
+            
+            self.logger.success(f"Browser closed.", tag="CDP")
+                
+        except Exception as e:
+            self.logger.error(f"Error launching standalone browser: {str(e)}", tag="CDP")
+            await managed_browser.cleanup()
+            return None
+        finally:
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            
+            # Make sure browser is fully cleaned up
+            await managed_browser.cleanup()
+        
+        # Return the CDP URL
+        return cdp_url
+    
+

crawl4ai/cli.py

@@ -1,5 +1,6 @@
 import click
 import os
+import sys
 import time
 
 import humanize
@@ -24,7 +25,6 @@ from crawl4ai import (
     BM25ContentFilter, 
     PruningContentFilter,
     BrowserProfiler,
-    DefaultMarkdownGenerator,
     LLMConfig
 )
 from litellm import completion
@@ -199,7 +199,24 @@ def show_examples():
     # 2. Then use that profile to crawl the authenticated site:
     crwl https://site-requiring-login.com/dashboard -p my-profile-name
 
-5️⃣  Sample Config Files:
+5️⃣  CDP Mode for Browser Automation:
+    # Launch browser with CDP debugging on default port 9222
+    crwl cdp
+
+    # Use a specific profile and custom port
+    crwl cdp -p my-profile -P 9223
+
+    # Launch headless browser with CDP enabled
+    crwl cdp --headless
+
+    # Launch in incognito mode (ignores profile)
+    crwl cdp --incognito
+
+    # Use the CDP URL with other tools (Puppeteer, Playwright, etc.)
+    # The URL will be displayed in the terminal when the browser starts
+
+    
+6️⃣  Sample Config Files:
 
 browser.yml:
     headless: true
@@ -257,7 +274,7 @@ llm_schema.json:
       }
     }
 
-6️⃣  Advanced Usage:
+7️⃣  Advanced Usage:
     # Combine configs with direct parameters
     crwl https://example.com -B browser.yml -b "headless=false,viewport_width=1920"
 
@@ -283,7 +300,7 @@ llm_schema.json:
 
 For more documentation visit: https://github.com/unclecode/crawl4ai
 
-7️⃣  Q&A with LLM:
+8️⃣  Q&A with LLM:
     # Ask a question about the content
     crwl https://example.com -q "What is the main topic discussed?"
 
@@ -311,7 +328,7 @@ For more documentation visit: https://github.com/unclecode/crawl4ai
     
     See full list of providers: https://docs.litellm.ai/docs/providers
 
-8️⃣ Profile Management:
+9️⃣ Profile Management:
     # Launch interactive profile manager
     crwl profiles
 
@@ -550,11 +567,89 @@ async def manage_profiles():
         # Add a separator between operations
         console.print("\n")
 
+
+
 @click.group(context_settings={"help_option_names": ["-h", "--help"]})
 def cli():
     """Crawl4AI CLI - Web content extraction and browser profile management tool"""
     pass
 
+
+@cli.command("cdp")
+@click.option("--user-data-dir", "-d", help="Directory to use for browser data (will be created if it doesn't exist)")
+@click.option("--port", "-P", type=int, default=9222, help="Debugging port (default: 9222)")
+@click.option("--browser-type", "-b", type=click.Choice(["chromium", "firefox"]), default="chromium", 
+              help="Browser type (default: chromium)")
+@click.option("--headless", is_flag=True, help="Run browser in headless mode")
+@click.option("--incognito", is_flag=True, help="Run in incognito/private mode (ignores user-data-dir)")
+def cdp_cmd(user_data_dir: Optional[str], port: int, browser_type: str, headless: bool, incognito: bool):
+    """Launch a standalone browser with CDP debugging enabled
+    
+    This command launches a browser with Chrome DevTools Protocol (CDP) debugging enabled,
+    prints the CDP URL, and keeps the browser running until you press 'q'.
+    
+    The CDP URL can be used for various automation and debugging tasks.
+    
+    Examples:
+        # Launch Chromium with CDP on default port 9222
+        crwl cdp
+        
+        # Use a specific directory for browser data and custom port
+        crwl cdp --user-data-dir ~/browser-data --port 9223
+        
+        # Launch in headless mode
+        crwl cdp --headless
+        
+        # Launch in incognito mode (ignores user-data-dir)
+        crwl cdp --incognito
+    """
+    profiler = BrowserProfiler()
+    
+    try:
+        # Handle data directory
+        data_dir = None
+        if not incognito and user_data_dir:
+            # Expand user path (~/something)
+            expanded_path = os.path.expanduser(user_data_dir)
+            
+            # Create directory if it doesn't exist
+            if not os.path.exists(expanded_path):
+                console.print(f"[yellow]Directory '{expanded_path}' doesn't exist. Creating it.[/yellow]")
+                os.makedirs(expanded_path, exist_ok=True)
+            
+            data_dir = expanded_path
+        
+        # Print launch info
+        console.print(Panel(
+            f"[cyan]Launching browser with CDP debugging[/cyan]\n\n"
+            f"Browser type: [green]{browser_type}[/green]\n"
+            f"Debugging port: [yellow]{port}[/yellow]\n"
+            f"User data directory: [cyan]{data_dir or 'Temporary directory'}[/cyan]\n"
+            f"Headless: [cyan]{'Yes' if headless else 'No'}[/cyan]\n"
+            f"Incognito: [cyan]{'Yes' if incognito else 'No'}[/cyan]\n\n"
+            f"[yellow]Press 'q' to quit when done[/yellow]",
+            title="CDP Browser",
+            border_style="cyan"
+        ))
+        
+        # Run the browser
+        cdp_url = anyio.run(
+            profiler.launch_standalone_browser,
+            browser_type,
+            data_dir,
+            port,
+            headless
+        )
+        
+        if not cdp_url:
+            console.print("[red]Failed to launch browser or get CDP URL[/red]")
+            sys.exit(1)
+            
+    except Exception as e:
+        console.print(f"[red]Error launching CDP browser: {str(e)}[/red]")
+        sys.exit(1)
+
+
 @cli.command("crawl")
 @click.argument("url", required=True)
 @click.option("--browser-config", "-B", type=click.Path(exists=True), help="Browser config file (YAML/JSON)")
@@ -615,28 +710,17 @@ def crawl_cmd(url: str, browser_config: str, crawler_config: str, filter_config:
             crawler_cfg = crawler_cfg.clone(**crawler)
             
         # Handle content filter config
-        if filter_config or output in ["markdown-fit", "md-fit"]:
-            if filter_config:
-                filter_conf = load_config_file(filter_config)
-            elif not filter_config and output in ["markdown-fit", "md-fit"]:
-                filter_conf = {
-                    "type": "pruning",
-                    "query": "",
-                    "threshold": 0.48
-                }
+        if filter_config:
+            filter_conf = load_config_file(filter_config)
             if filter_conf["type"] == "bm25":
-                crawler_cfg.markdown_generator = DefaultMarkdownGenerator(
-                    content_filter = BM25ContentFilter(
-                        user_query=filter_conf.get("query"),
-                        bm25_threshold=filter_conf.get("threshold", 1.0)
-                    )
+                crawler_cfg.content_filter = BM25ContentFilter(
+                    user_query=filter_conf.get("query"),
+                    bm25_threshold=filter_conf.get("threshold", 1.0)
                 )
             elif filter_conf["type"] == "pruning":
-                crawler_cfg.markdown_generator = DefaultMarkdownGenerator(
-                    content_filter = PruningContentFilter(
-                        user_query=filter_conf.get("query"),
-                        threshold=filter_conf.get("threshold", 0.48)
-                    )
+                crawler_cfg.content_filter = PruningContentFilter(
+                    user_query=filter_conf.get("query"),
+                    threshold=filter_conf.get("threshold", 0.48)
                 )
                 
         # Handle extraction strategy
@@ -749,6 +833,7 @@ def default(url: str, example: bool, browser_config: str, crawler_config: str, f
     Other commands:
         crwl profiles   - Manage browser profiles for identity-based crawling
         crwl crawl      - Crawl a website with advanced options
+        crwl cdp        - Launch browser with CDP debugging enabled
         crwl examples   - Show more usage examples
     """
 

crawl4ai/configs/__init__.py

@@ -1,2 +0,0 @@
-from .proxy_config import ProxyConfig
-__all__ = ["ProxyConfig"]

crawl4ai/configs/proxy_config.py

@@ -1,113 +0,0 @@
-import os
-from typing import Dict, List, Optional
-
-
-class ProxyConfig:
-    def __init__(
-        self,
-        server: str,
-        username: Optional[str] = None,
-        password: Optional[str] = None,
-        ip: Optional[str] = None,
-    ):
-        """Configuration class for a single proxy.
-        
-        Args:
-            server: Proxy server URL (e.g., "http://127.0.0.1:8080")
-            username: Optional username for proxy authentication
-            password: Optional password for proxy authentication
-            ip: Optional IP address for verification purposes
-        """
-        self.server = server
-        self.username = username
-        self.password = password
-        
-        # Extract IP from server if not explicitly provided
-        self.ip = ip or self._extract_ip_from_server()
-    
-    def _extract_ip_from_server(self) -> Optional[str]:
-        """Extract IP address from server URL."""
-        try:
-            # Simple extraction assuming http://ip:port format
-            if "://" in self.server:
-                parts = self.server.split("://")[1].split(":")
-                return parts[0]
-            else:
-                parts = self.server.split(":")
-                return parts[0]
-        except Exception:
-            return None
-    
-    @staticmethod
-    def from_string(proxy_str: str) -> "ProxyConfig":
-        """Create a ProxyConfig from a string in the format 'ip:port:username:password'."""
-        parts = proxy_str.split(":")
-        if len(parts) == 4:  # ip:port:username:password
-            ip, port, username, password = parts
-            return ProxyConfig(
-                server=f"http://{ip}:{port}",
-                username=username,
-                password=password,
-                ip=ip
-            )
-        elif len(parts) == 2:  # ip:port only
-            ip, port = parts
-            return ProxyConfig(
-                server=f"http://{ip}:{port}",
-                ip=ip
-            )
-        else:
-            raise ValueError(f"Invalid proxy string format: {proxy_str}")
-    
-    @staticmethod
-    def from_dict(proxy_dict: Dict) -> "ProxyConfig":
-        """Create a ProxyConfig from a dictionary."""
-        return ProxyConfig(
-            server=proxy_dict.get("server"),
-            username=proxy_dict.get("username"),
-            password=proxy_dict.get("password"),
-            ip=proxy_dict.get("ip")
-        )
-    
-    @staticmethod
-    def from_env(env_var: str = "PROXIES") -> List["ProxyConfig"]:
-        """Load proxies from environment variable.
-        
-        Args:
-            env_var: Name of environment variable containing comma-separated proxy strings
-            
-        Returns:
-            List of ProxyConfig objects
-        """
-        proxies = []
-        try:
-            proxy_list = os.getenv(env_var, "").split(",")
-            for proxy in proxy_list:
-                if not proxy:
-                    continue
-                proxies.append(ProxyConfig.from_string(proxy))
-        except Exception as e:
-            print(f"Error loading proxies from environment: {e}")
-        return proxies
-    
-    def to_dict(self) -> Dict:
-        """Convert to dictionary representation."""
-        return {
-            "server": self.server,
-            "username": self.username,
-            "password": self.password,
-            "ip": self.ip
-        }
-    
-    def clone(self, **kwargs) -> "ProxyConfig":
-        """Create a copy of this configuration with updated values.
-
-        Args:
-            **kwargs: Key-value pairs of configuration options to update
-
-        Returns:
-            ProxyConfig: A new instance with the specified updates
-        """
-        config_dict = self.to_dict()
-        config_dict.update(kwargs)
-        return ProxyConfig.from_dict(config_dict)

crawl4ai/content_scraping_strategy.py

@@ -155,6 +155,7 @@ class WebScrapingStrategy(ContentScrapingStrategy):
                 for aud in raw_result.get("media", {}).get("audios", [])
                 if aud
             ],
+            tables=raw_result.get("media", {}).get("tables", [])
         )
 
         # Convert links
@@ -193,6 +194,153 @@ class WebScrapingStrategy(ContentScrapingStrategy):
         """
         return await asyncio.to_thread(self._scrap, url, html, **kwargs)
 
+    def is_data_table(self, table: Tag, **kwargs) -> bool:
+        """
+        Determine if a table element is a data table (not a layout table).
+
+        Args:
+            table (Tag): BeautifulSoup Tag representing a table element
+            **kwargs: Additional keyword arguments including table_score_threshold
+
+        Returns:
+            bool: True if the table is a data table, False otherwise
+        """
+        score = 0
+        
+        # Check for thead and tbody
+        has_thead = len(table.select('thead')) > 0
+        has_tbody = len(table.select('tbody')) > 0
+        if has_thead:
+            score += 2
+        if has_tbody:
+            score += 1
+            
+        # Check for th elements
+        th_count = len(table.select('th'))
+        if th_count > 0:
+            score += 2
+            if has_thead or len(table.select('tr:first-child th')) > 0:
+                score += 1
+                
+        # Check for nested tables
+        if len(table.select('table')) > 0:
+            score -= 3
+            
+        # Role attribute check
+        role = table.get('role', '').lower()
+        if role in {'presentation', 'none'}:
+            score -= 3
+            
+        # Column consistency
+        rows = table.select('tr')
+        if not rows:
+            return False
+            
+        col_counts = [len(row.select('td, th')) for row in rows]
+        avg_cols = sum(col_counts) / len(col_counts)
+        variance = sum((c - avg_cols)**2 for c in col_counts) / len(col_counts)
+        if variance < 1:
+            score += 2
+            
+        # Caption and summary
+        if table.select('caption'):
+            score += 2
+        if table.has_attr('summary') and table['summary']:
+            score += 1
+            
+        # Text density
+        total_text = sum(len(cell.get_text().strip()) for row in rows for cell in row.select('td, th'))
+        total_tags = sum(1 for _ in table.descendants if isinstance(_, Tag))
+        text_ratio = total_text / (total_tags + 1e-5)
+        if text_ratio > 20:
+            score += 3
+        elif text_ratio > 10:
+            score += 2
+            
+        # Data attributes
+        data_attrs = sum(1 for attr in table.attrs if attr.startswith('data-'))
+        score += data_attrs * 0.5
+        
+        # Size check
+        if avg_cols >= 2 and len(rows) >= 2:
+            score += 2
+            
+        threshold = kwargs.get('table_score_threshold', 7)
+        return score >= threshold
+    
+    def extract_table_data(self, table: Tag) -> dict:
+        """
+        Extract structured data from a table element.
+        
+        Args:
+            table (Tag): BeautifulSoup Tag representing a table element
+            
+        Returns:
+            dict: Dictionary containing table data (headers, rows, caption, summary)
+        """
+        caption_elem = table.select_one('caption')
+        caption = caption_elem.get_text().strip() if caption_elem else ""
+        summary = table.get('summary', '').strip()
+        
+        # Extract headers with colspan handling
+        headers = []
+        thead_rows = table.select('thead tr')
+        if thead_rows:
+            header_cells = thead_rows[0].select('th')
+            for cell in header_cells:
+                text = cell.get_text().strip()
+                colspan = int(cell.get('colspan', 1))
+                headers.extend([text] * colspan)
+        else:
+            first_row = table.select('tr:first-child')
+            if first_row:
+                for cell in first_row[0].select('th, td'):
+                    text = cell.get_text().strip()
+                    colspan = int(cell.get('colspan', 1))
+                    headers.extend([text] * colspan)
+        
+        # Extract rows with colspan handling
+        rows = []
+        all_rows = table.select('tr')
+        thead = table.select_one('thead')
+        tbody_rows = []
+
+        if thead:
+            thead_rows = thead.select('tr')
+            tbody_rows = [row for row in all_rows if row not in thead_rows]
+        else:
+            if all_rows and all_rows[0].select('th'):
+                tbody_rows = all_rows[1:]
+            else:
+                tbody_rows = all_rows
+                
+        for row in tbody_rows:        
+        # for row in table.select('tr:not(:has(ancestor::thead))'):
+            row_data = []
+            for cell in row.select('td'):
+                text = cell.get_text().strip()
+                colspan = int(cell.get('colspan', 1))
+                row_data.extend([text] * colspan)
+            if row_data:
+                rows.append(row_data)
+                
+        # Align rows with headers
+        max_columns = len(headers) if headers else (max(len(row) for row in rows) if rows else 0)
+        aligned_rows = []
+        for row in rows:
+            aligned = row[:max_columns] + [''] * (max_columns - len(row))
+            aligned_rows.append(aligned)
+            
+        if not headers:
+            headers = [f"Column {i+1}" for i in range(max_columns)]
+            
+        return {
+            "headers": headers,
+            "rows": aligned_rows,
+            "caption": caption,
+            "summary": summary,
+        }
+    
     def flatten_nested_elements(self, node):
         """
         Flatten nested elements in a HTML tree.
@@ -431,7 +579,7 @@ class WebScrapingStrategy(ContentScrapingStrategy):
         Returns:
             dict: A dictionary containing the processed element information.
         """
-        media = {"images": [], "videos": [], "audios": []}
+        media = {"images": [], "videos": [], "audios": [], "tables": []}
         internal_links_dict = {}
         external_links_dict = {}
         self._process_element(
@@ -688,6 +836,7 @@ class WebScrapingStrategy(ContentScrapingStrategy):
         html: str,
         word_count_threshold: int = MIN_WORD_THRESHOLD,
         css_selector: str = None,
+        target_elements: List[str] = None,
         **kwargs,
     ) -> Dict[str, Any]:
         """
@@ -742,22 +891,37 @@ class WebScrapingStrategy(ContentScrapingStrategy):
                 for element in body.select(excluded_selector):
                     element.extract()
 
-        if css_selector:
-            selected_elements = body.select(css_selector)
-            if not selected_elements:
-                return {
-                    "markdown": "",
-                    "cleaned_html": "",
-                    "success": True,
-                    "media": {"images": [], "videos": [], "audios": []},
-                    "links": {"internal": [], "external": []},
-                    "metadata": {},
-                    "message": f"No elements found for CSS selector: {css_selector}",
-                }
-                # raise InvalidCSSSelectorError(f"Invalid CSS selector, No elements found for CSS selector: {css_selector}")
-            body = soup.new_tag("div")
-            for el in selected_elements:
-                body.append(el)
+        # if False and css_selector:
+        #     selected_elements = body.select(css_selector)
+        #     if not selected_elements:
+        #         return {
+        #             "markdown": "",
+        #             "cleaned_html": "",
+        #             "success": True,
+        #             "media": {"images": [], "videos": [], "audios": []},
+        #             "links": {"internal": [], "external": []},
+        #             "metadata": {},
+        #             "message": f"No elements found for CSS selector: {css_selector}",
+        #         }
+        #         # raise InvalidCSSSelectorError(f"Invalid CSS selector, No elements found for CSS selector: {css_selector}")
+        #     body = soup.new_tag("div")
+        #     for el in selected_elements:
+        #         body.append(el)
+
+        content_element = None
+        if target_elements:
+            try:
+                for_content_targeted_element = []
+                for target_element in target_elements:
+                    for_content_targeted_element.extend(body.select(target_element))
+                content_element = soup.new_tag("div")
+                for el in for_content_targeted_element:
+                    content_element.append(el)
+            except Exception as e:
+                self._log("error", f"Error with target element detection: {str(e)}", "SCRAPE")
+                return None
+        else:
+            content_element = body        
 
         kwargs["exclude_social_media_domains"] = set(
             kwargs.get("exclude_social_media_domains", []) + SOCIAL_MEDIA_DOMAINS
@@ -797,6 +961,15 @@ class WebScrapingStrategy(ContentScrapingStrategy):
             if result is not None
             for img in result
         ]
+        
+        # Process tables if not excluded
+        excluded_tags = set(kwargs.get("excluded_tags", []) or [])
+        if 'table' not in excluded_tags:
+            tables = body.find_all('table')
+            for table in tables:
+                if self.is_data_table(table, **kwargs):
+                    table_data = self.extract_table_data(table)
+                    media["tables"].append(table_data)
 
         body = self.flatten_nested_elements(body)
         base64_pattern = re.compile(r'data:image/[^;]+;base64,([^"]+)')
@@ -808,7 +981,7 @@ class WebScrapingStrategy(ContentScrapingStrategy):
 
         str_body = ""
         try:
-            str_body = body.encode_contents().decode("utf-8")
+            str_body = content_element.encode_contents().decode("utf-8")
         except Exception:
             # Reset body to the original HTML
             success = False
@@ -847,7 +1020,6 @@ class WebScrapingStrategy(ContentScrapingStrategy):
         cleaned_html = str_body.replace("\n\n", "\n").replace("  ", " ")
 
         return {
-            # **markdown_content,
             "cleaned_html": cleaned_html,
             "success": success,
             "media": media,
@@ -1187,12 +1359,125 @@ class LXMLWebScrapingStrategy(WebScrapingStrategy):
 
         return root
 
+    def is_data_table(self, table: etree.Element, **kwargs) -> bool:
+        score = 0
+        # Check for thead and tbody
+        has_thead = len(table.xpath(".//thead")) > 0
+        has_tbody = len(table.xpath(".//tbody")) > 0
+        if has_thead:
+            score += 2
+        if has_tbody:
+            score += 1
+
+        # Check for th elements
+        th_count = len(table.xpath(".//th"))
+        if th_count > 0:
+            score += 2
+            if has_thead or table.xpath(".//tr[1]/th"):
+                score += 1
+
+        # Check for nested tables
+        if len(table.xpath(".//table")) > 0:
+            score -= 3
+
+        # Role attribute check
+        role = table.get("role", "").lower()
+        if role in {"presentation", "none"}:
+            score -= 3
+
+        # Column consistency
+        rows = table.xpath(".//tr")
+        if not rows:
+            return False
+        col_counts = [len(row.xpath(".//td|.//th")) for row in rows]
+        avg_cols = sum(col_counts) / len(col_counts)
+        variance = sum((c - avg_cols)**2 for c in col_counts) / len(col_counts)
+        if variance < 1:
+            score += 2
+
+        # Caption and summary
+        if table.xpath(".//caption"):
+            score += 2
+        if table.get("summary"):
+            score += 1
+
+        # Text density
+        total_text = sum(len(''.join(cell.itertext()).strip()) for row in rows for cell in row.xpath(".//td|.//th"))
+        total_tags = sum(1 for _ in table.iterdescendants())
+        text_ratio = total_text / (total_tags + 1e-5)
+        if text_ratio > 20:
+            score += 3
+        elif text_ratio > 10:
+            score += 2
+
+        # Data attributes
+        data_attrs = sum(1 for attr in table.attrib if attr.startswith('data-'))
+        score += data_attrs * 0.5
+
+        # Size check
+        if avg_cols >= 2 and len(rows) >= 2:
+            score += 2
+
+        threshold = kwargs.get("table_score_threshold", 7)
+        return score >= threshold
+
+    def extract_table_data(self, table: etree.Element) -> dict:
+        caption = table.xpath(".//caption/text()")
+        caption = caption[0].strip() if caption else ""
+        summary = table.get("summary", "").strip()
+
+        # Extract headers with colspan handling
+        headers = []
+        thead_rows = table.xpath(".//thead/tr")
+        if thead_rows:
+            header_cells = thead_rows[0].xpath(".//th")
+            for cell in header_cells:
+                text = cell.text_content().strip()
+                colspan = int(cell.get("colspan", 1))
+                headers.extend([text] * colspan)
+        else:
+            first_row = table.xpath(".//tr[1]")
+            if first_row:
+                for cell in first_row[0].xpath(".//th|.//td"):
+                    text = cell.text_content().strip()
+                    colspan = int(cell.get("colspan", 1))
+                    headers.extend([text] * colspan)
+
+        # Extract rows with colspan handling
+        rows = []
+        for row in table.xpath(".//tr[not(ancestor::thead)]"):
+            row_data = []
+            for cell in row.xpath(".//td"):
+                text = cell.text_content().strip()
+                colspan = int(cell.get("colspan", 1))
+                row_data.extend([text] * colspan)
+            if row_data:
+                rows.append(row_data)
+
+        # Align rows with headers
+        max_columns = len(headers) if headers else (max(len(row) for row in rows) if rows else 0)
+        aligned_rows = []
+        for row in rows:
+            aligned = row[:max_columns] + [''] * (max_columns - len(row))
+            aligned_rows.append(aligned)
+
+        if not headers:
+            headers = [f"Column {i+1}" for i in range(max_columns)]
+
+        return {
+            "headers": headers,
+            "rows": aligned_rows,
+            "caption": caption,
+            "summary": summary,
+        }
+
     def _scrap(
         self,
         url: str,
         html: str,
         word_count_threshold: int = MIN_WORD_THRESHOLD,
         css_selector: str = None,
+        target_elements: List[str] = None,
         **kwargs,
     ) -> Dict[str, Any]:
         if not html:
@@ -1243,24 +1528,38 @@ class LXMLWebScrapingStrategy(WebScrapingStrategy):
                 meta = {}
 
             # Handle CSS selector targeting
-            if css_selector:
+            # if css_selector:
+            #     try:
+            #         selected_elements = body.cssselect(css_selector)
+            #         if not selected_elements:
+            #             return {
+            #                 "markdown": "",
+            #                 "cleaned_html": "",
+            #                 "success": True,
+            #                 "media": {"images": [], "videos": [], "audios": []},
+            #                 "links": {"internal": [], "external": []},
+            #                 "metadata": meta,
+            #                 "message": f"No elements found for CSS selector: {css_selector}",
+            #             }
+            #         body = lhtml.Element("div")
+            #         body.extend(selected_elements)
+            #     except Exception as e:
+            #         self._log("error", f"Error with CSS selector: {str(e)}", "SCRAPE")
+            #         return None
+
+            content_element = None
+            if target_elements:
                 try:
-                    selected_elements = body.cssselect(css_selector)
-                    if not selected_elements:
-                        return {
-                            "markdown": "",
-                            "cleaned_html": "",
-                            "success": True,
-                            "media": {"images": [], "videos": [], "audios": []},
-                            "links": {"internal": [], "external": []},
-                            "metadata": meta,
-                            "message": f"No elements found for CSS selector: {css_selector}",
-                        }
-                    body = lhtml.Element("div")
-                    body.extend(selected_elements)
+                    for_content_targeted_element = []
+                    for target_element in target_elements:
+                        for_content_targeted_element.extend(body.cssselect(target_element))
+                    content_element = lhtml.Element("div")
+                    content_element.extend(for_content_targeted_element)
                 except Exception as e:
-                    self._log("error", f"Error with CSS selector: {str(e)}", "SCRAPE")
+                    self._log("error", f"Error with target element detection: {str(e)}", "SCRAPE")
                     return None
+            else:
+                content_element = body
 
             # Remove script and style tags
             for tag in ["script", "style", "link", "meta", "noscript"]:
@@ -1284,7 +1583,7 @@ class LXMLWebScrapingStrategy(WebScrapingStrategy):
                         form.getparent().remove(form)
 
             # Process content
-            media = {"images": [], "videos": [], "audios": []}
+            media = {"images": [], "videos": [], "audios": [], "tables": []}
             internal_links_dict = {}
             external_links_dict = {}
 
@@ -1298,6 +1597,13 @@ class LXMLWebScrapingStrategy(WebScrapingStrategy):
                 **kwargs,
             )
 
+            if 'table' not in excluded_tags:
+                tables = body.xpath(".//table")
+                for table in tables:
+                    if self.is_data_table(table, **kwargs):
+                        table_data = self.extract_table_data(table)
+                        media["tables"].append(table_data)
+
             # Handle only_text option
             if kwargs.get("only_text", False):
                 for tag in ONLY_TEXT_ELIGIBLE_TAGS:
@@ -1324,7 +1630,8 @@ class LXMLWebScrapingStrategy(WebScrapingStrategy):
 
             # Generate output HTML
             cleaned_html = lhtml.tostring(
-                body,
+                # body,   
+                content_element,
                 encoding="unicode",
                 pretty_print=True,
                 method="html",
@@ -1369,7 +1676,12 @@ class LXMLWebScrapingStrategy(WebScrapingStrategy):
             return {
                 "cleaned_html": cleaned_html,
                 "success": False,
-                "media": {"images": [], "videos": [], "audios": []},
+                "media": {
+                    "images": [],
+                    "videos": [],
+                    "audios": [],
+                    "tables": []
+                },
                 "links": {"internal": [], "external": []},
                 "metadata": {},
             }

crawl4ai/deep_crawling/filters.py

@@ -124,6 +124,7 @@ class URLPatternFilter(URLFilter):
         "_simple_prefixes",
         "_domain_patterns",
         "_path_patterns",
+        "_reverse",
     )
 
     PATTERN_TYPES = {
@@ -138,8 +139,10 @@ class URLPatternFilter(URLFilter):
         self,
         patterns: Union[str, Pattern, List[Union[str, Pattern]]],
         use_glob: bool = True,
+        reverse: bool = False,
     ):
         super().__init__()
+        self._reverse = reverse
         patterns = [patterns] if isinstance(patterns, (str, Pattern)) else patterns
 
         self._simple_suffixes = set()
@@ -205,36 +208,40 @@ class URLPatternFilter(URLFilter):
 
     @lru_cache(maxsize=10000)
     def apply(self, url: str) -> bool:
-        """Hierarchical pattern matching"""
         # Quick suffix check (*.html)
         if self._simple_suffixes:
             path = url.split("?")[0]
             if path.split("/")[-1].split(".")[-1] in self._simple_suffixes:
-                self._update_stats(True)
-                return True
+                result = True
+                self._update_stats(result)
+                return not result if self._reverse else result
 
         # Domain check
         if self._domain_patterns:
             for pattern in self._domain_patterns:
                 if pattern.match(url):
-                    self._update_stats(True)
-                    return True
+                    result = True
+                    self._update_stats(result)
+                    return not result if self._reverse else result
 
         # Prefix check (/foo/*)
         if self._simple_prefixes:
             path = url.split("?")[0]
             if any(path.startswith(p) for p in self._simple_prefixes):
-                self._update_stats(True)
-                return True
+                result = True
+                self._update_stats(result)
+                return not result if self._reverse else result
 
         # Complex patterns
         if self._path_patterns:
             if any(p.search(url) for p in self._path_patterns):
-                self._update_stats(True)
-                return True
+                result = True
+                self._update_stats(result)
+                return not result if self._reverse else result
 
-        self._update_stats(False)
-        return False
+        result = False
+        self._update_stats(result)
+        return not result if self._reverse else result
 
 
 class ContentTypeFilter(URLFilter):

crawl4ai/extraction_strategy.py

@@ -1168,7 +1168,8 @@ class JsonCssExtractionStrategy(JsonElementExtractionStrategy):
         super().__init__(schema, **kwargs)
 
     def _parse_html(self, html_content: str):
-        return BeautifulSoup(html_content, "html.parser")
+        # return BeautifulSoup(html_content, "html.parser")
+        return BeautifulSoup(html_content, "lxml")
 
     def _get_base_elements(self, parsed_html, selector: str):
         return parsed_html.select(selector)
@@ -1187,6 +1188,373 @@ class JsonCssExtractionStrategy(JsonElementExtractionStrategy):
     def _get_element_attribute(self, element, attribute: str):
         return element.get(attribute)
 
+class JsonLxmlExtractionStrategy(JsonElementExtractionStrategy):
+    def __init__(self, schema: Dict[str, Any], **kwargs):
+        kwargs["input_format"] = "html"
+        super().__init__(schema, **kwargs)
+        self._selector_cache = {}
+        self._xpath_cache = {}
+        self._result_cache = {}
+        
+        # Control selector optimization strategy
+        self.use_caching = kwargs.get("use_caching", True)
+        self.optimize_common_patterns = kwargs.get("optimize_common_patterns", True)
+        
+        # Load lxml dependencies once
+        from lxml import etree, html
+        from lxml.cssselect import CSSSelector
+        self.etree = etree
+        self.html_parser = html
+        self.CSSSelector = CSSSelector
+    
+    def _parse_html(self, html_content: str):
+        """Parse HTML content with error recovery"""
+        try:
+            parser = self.etree.HTMLParser(recover=True, remove_blank_text=True)
+            return self.etree.fromstring(html_content, parser)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error parsing HTML, falling back to alternative method: {e}")
+            try:
+                return self.html_parser.fromstring(html_content)
+            except Exception as e2:
+                if self.verbose:
+                    print(f"Critical error parsing HTML: {e2}")
+                # Create minimal document as fallback
+                return self.etree.Element("html")
+    
+    def _optimize_selector(self, selector_str):
+        """Optimize common selector patterns for better performance"""
+        if not self.optimize_common_patterns:
+            return selector_str
+            
+        # Handle td:nth-child(N) pattern which is very common in table scraping
+        import re
+        if re.search(r'td:nth-child\(\d+\)', selector_str):
+            return selector_str  # Already handled specially in _apply_selector
+            
+        # Split complex selectors into parts for optimization
+        parts = selector_str.split()
+        if len(parts) <= 1:
+            return selector_str
+            
+        # For very long selectors, consider using just the last specific part
+        if len(parts) > 3 and any(p.startswith('.') or p.startswith('#') for p in parts):
+            specific_parts = [p for p in parts if p.startswith('.') or p.startswith('#')]
+            if specific_parts:
+                return specific_parts[-1]  # Use most specific class/id selector
+                
+        return selector_str
+    
+    def _create_selector_function(self, selector_str):
+        """Create a selector function that handles all edge cases"""
+        original_selector = selector_str
+        
+        # Try to optimize the selector if appropriate
+        if self.optimize_common_patterns:
+            selector_str = self._optimize_selector(selector_str)
+        
+        try:
+            # Attempt to compile the CSS selector
+            compiled = self.CSSSelector(selector_str)
+            xpath = compiled.path
+            
+            # Store XPath for later use
+            self._xpath_cache[selector_str] = xpath
+            
+            # Create the wrapper function that implements the selection strategy
+            def selector_func(element, context_sensitive=True):
+                cache_key = None
+                
+                # Use result caching if enabled
+                if self.use_caching:
+                    # Create a cache key based on element and selector
+                    element_id = element.get('id', '') or str(hash(element))
+                    cache_key = f"{element_id}::{selector_str}"
+                    
+                    if cache_key in self._result_cache:
+                        return self._result_cache[cache_key]
+                
+                results = []
+                try:
+                    # Strategy 1: Direct CSS selector application (fastest)
+                    results = compiled(element)
+                    
+                    # If that fails and we need context sensitivity
+                    if not results and context_sensitive:
+                        # Strategy 2: Try XPath with context adjustment
+                        context_xpath = self._make_context_sensitive_xpath(xpath, element)
+                        if context_xpath:
+                            results = element.xpath(context_xpath)
+                        
+                        # Strategy 3: Handle special case - nth-child
+                        if not results and 'nth-child' in original_selector:
+                            results = self._handle_nth_child_selector(element, original_selector)
+                        
+                        # Strategy 4: Direct descendant search for class/ID selectors
+                        if not results:
+                            results = self._fallback_class_id_search(element, original_selector)
+                            
+                        # Strategy 5: Last resort - tag name search for the final part
+                        if not results:
+                            parts = original_selector.split()
+                            if parts:
+                                last_part = parts[-1]
+                                # Extract tag name from the selector
+                                tag_match = re.match(r'^(\w+)', last_part)
+                                if tag_match:
+                                    tag_name = tag_match.group(1)
+                                    results = element.xpath(f".//{tag_name}")
+                    
+                    # Cache results if caching is enabled
+                    if self.use_caching and cache_key:
+                        self._result_cache[cache_key] = results
+                        
+                except Exception as e:
+                    if self.verbose:
+                        print(f"Error applying selector '{selector_str}': {e}")
+                
+                return results
+                
+            return selector_func
+            
+        except Exception as e:
+            if self.verbose:
+                print(f"Error compiling selector '{selector_str}': {e}")
+            
+            # Fallback function for invalid selectors
+            return lambda element, context_sensitive=True: []
+    
+    def _make_context_sensitive_xpath(self, xpath, element):
+        """Convert absolute XPath to context-sensitive XPath"""
+        try:
+            # If starts with descendant-or-self, it's already context-sensitive
+            if xpath.startswith('descendant-or-self::'):
+                return xpath
+                
+            # Remove leading slash if present
+            if xpath.startswith('/'):
+                context_xpath = f".{xpath}"
+            else:
+                context_xpath = f".//{xpath}"
+                
+            # Validate the XPath by trying it
+            try:
+                element.xpath(context_xpath)
+                return context_xpath
+            except:
+                # If that fails, try a simpler descendant search
+                return f".//{xpath.split('/')[-1]}"
+        except:
+            return None
+    
+    def _handle_nth_child_selector(self, element, selector_str):
+        """Special handling for nth-child selectors in tables"""
+        import re
+        results = []
+        
+        try:
+            # Extract the column number from td:nth-child(N)
+            match = re.search(r'td:nth-child\((\d+)\)', selector_str)
+            if match:
+                col_num = match.group(1)
+                
+                # Check if there's content after the nth-child part
+                remaining_selector = selector_str.split(f"td:nth-child({col_num})", 1)[-1].strip()
+                
+                if remaining_selector:
+                    # If there's a specific element we're looking for after the column
+                    # Extract any tag names from the remaining selector
+                    tag_match = re.search(r'(\w+)', remaining_selector)
+                    tag_name = tag_match.group(1) if tag_match else '*'
+                    results = element.xpath(f".//td[{col_num}]//{tag_name}")
+                else:
+                    # Just get the column cell
+                    results = element.xpath(f".//td[{col_num}]")
+        except Exception as e:
+            if self.verbose:
+                print(f"Error handling nth-child selector: {e}")
+                
+        return results
+    
+    def _fallback_class_id_search(self, element, selector_str):
+        """Fallback to search by class or ID"""
+        results = []
+        
+        try:
+            # Extract class selectors (.classname)
+            import re
+            class_matches = re.findall(r'\.([a-zA-Z0-9_-]+)', selector_str)
+            
+            # Extract ID selectors (#idname)
+            id_matches = re.findall(r'#([a-zA-Z0-9_-]+)', selector_str)
+            
+            # Try each class
+            for class_name in class_matches:
+                class_results = element.xpath(f".//*[contains(@class, '{class_name}')]")
+                results.extend(class_results)
+                
+            # Try each ID (usually more specific)
+            for id_name in id_matches:
+                id_results = element.xpath(f".//*[@id='{id_name}']")
+                results.extend(id_results)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error in fallback class/id search: {e}")
+                
+        return results
+    
+    def _get_selector(self, selector_str):
+        """Get or create a selector function with caching"""
+        if selector_str not in self._selector_cache:
+            self._selector_cache[selector_str] = self._create_selector_function(selector_str)
+        return self._selector_cache[selector_str]
+    
+    def _get_base_elements(self, parsed_html, selector: str):
+        """Get all base elements using the selector"""
+        selector_func = self._get_selector(selector)
+        # For base elements, we don't need context sensitivity
+        return selector_func(parsed_html, context_sensitive=False)
+    
+    def _get_elements(self, element, selector: str):
+        """Get child elements using the selector with context sensitivity"""
+        selector_func = self._get_selector(selector)
+        return selector_func(element, context_sensitive=True)
+    
+    def _get_element_text(self, element) -> str:
+        """Extract normalized text from element"""
+        try:
+            # Get all text nodes and normalize
+            text = " ".join(t.strip() for t in element.xpath(".//text()") if t.strip())
+            return text
+        except Exception as e:
+            if self.verbose:
+                print(f"Error extracting text: {e}")
+            # Fallback
+            try:
+                return element.text_content().strip()
+            except:
+                return ""
+    
+    def _get_element_html(self, element) -> str:
+        """Get HTML string representation of element"""
+        try:
+            return self.etree.tostring(element, encoding='unicode', method='html')
+        except Exception as e:
+            if self.verbose:
+                print(f"Error serializing HTML: {e}")
+            return ""
+    
+    def _get_element_attribute(self, element, attribute: str):
+        """Get attribute value safely"""
+        try:
+            return element.get(attribute)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error getting attribute '{attribute}': {e}")
+            return None
+            
+    def _clear_caches(self):
+        """Clear caches to free memory"""
+        if self.use_caching:
+            self._result_cache.clear()
+
+class JsonLxmlExtractionStrategy_naive(JsonElementExtractionStrategy):
+    def __init__(self, schema: Dict[str, Any], **kwargs):
+        kwargs["input_format"] = "html"  # Force HTML input
+        super().__init__(schema, **kwargs)
+        self._selector_cache = {}
+    
+    def _parse_html(self, html_content: str):
+        from lxml import etree
+        parser = etree.HTMLParser(recover=True)
+        return etree.fromstring(html_content, parser)
+    
+    def _get_selector(self, selector_str):
+        """Get a selector function that works within the context of an element"""
+        if selector_str not in self._selector_cache:
+            from lxml.cssselect import CSSSelector
+            try:
+                # Store both the compiled selector and its xpath translation
+                compiled = CSSSelector(selector_str)
+                
+                # Create a function that will apply this selector appropriately
+                def select_func(element):
+                    try:
+                        # First attempt: direct CSS selector application
+                        results = compiled(element)
+                        if results:
+                            return results
+                        
+                        # Second attempt: contextual XPath selection
+                        # Convert the root-based XPath to a context-based XPath
+                        xpath = compiled.path
+                        
+                        # If the XPath already starts with descendant-or-self, handle it specially
+                        if xpath.startswith('descendant-or-self::'):
+                            context_xpath = xpath
+                        else:
+                            # For normal XPath expressions, make them relative to current context
+                            context_xpath = f"./{xpath.lstrip('/')}"
+                        
+                        results = element.xpath(context_xpath)
+                        if results:
+                            return results
+                        
+                        # Final fallback: simple descendant search for common patterns
+                        if 'nth-child' in selector_str:
+                            # Handle td:nth-child(N) pattern
+                            import re
+                            match = re.search(r'td:nth-child\((\d+)\)', selector_str)
+                            if match:
+                                col_num = match.group(1)
+                                sub_selector = selector_str.split(')', 1)[-1].strip()
+                                if sub_selector:
+                                    return element.xpath(f".//td[{col_num}]//{sub_selector}")
+                                else:
+                                    return element.xpath(f".//td[{col_num}]")
+                        
+                        # Last resort: try each part of the selector separately
+                        parts = selector_str.split()
+                        if len(parts) > 1 and parts[-1]:
+                            return element.xpath(f".//{parts[-1]}")
+                            
+                        return []
+                    except Exception as e:
+                        if self.verbose:
+                            print(f"Error applying selector '{selector_str}': {e}")
+                        return []
+                
+                self._selector_cache[selector_str] = select_func
+            except Exception as e:
+                if self.verbose:
+                    print(f"Error compiling selector '{selector_str}': {e}")
+                
+                # Fallback function for invalid selectors
+                def fallback_func(element):
+                    return []
+                
+                self._selector_cache[selector_str] = fallback_func
+                
+        return self._selector_cache[selector_str]
+    
+    def _get_base_elements(self, parsed_html, selector: str):
+        selector_func = self._get_selector(selector)
+        return selector_func(parsed_html)
+    
+    def _get_elements(self, element, selector: str):
+        selector_func = self._get_selector(selector)
+        return selector_func(element)
+    
+    def _get_element_text(self, element) -> str:
+        return "".join(element.xpath(".//text()")).strip()
+    
+    def _get_element_html(self, element) -> str:
+        from lxml import etree
+        return etree.tostring(element, encoding='unicode')
+    
+    def _get_element_attribute(self, element, attribute: str):
+        return element.get(attribute)    
 
 class JsonXPathExtractionStrategy(JsonElementExtractionStrategy):
     """

crawl4ai/models.py

@@ -326,6 +326,7 @@ class Media(BaseModel):
     audios: List[
         MediaItem
     ] = []  # Using MediaItem model for now, can be extended with Audio model if needed
+    tables: List[Dict] = []  # Table data extracted from HTML tables
 
 
 class Links(BaseModel):

crawl4ai/proxy_strategy.py

@@ -1,8 +1,119 @@
 from typing import List, Dict, Optional
 from abc import ABC, abstractmethod
 from itertools import cycle
+import os
+
+
+class ProxyConfig:
+    def __init__(
+        self,
+        server: str,
+        username: Optional[str] = None,
+        password: Optional[str] = None,
+        ip: Optional[str] = None,
+    ):
+        """Configuration class for a single proxy.
+        
+        Args:
+            server: Proxy server URL (e.g., "http://127.0.0.1:8080")
+            username: Optional username for proxy authentication
+            password: Optional password for proxy authentication
+            ip: Optional IP address for verification purposes
+        """
+        self.server = server
+        self.username = username
+        self.password = password
+        
+        # Extract IP from server if not explicitly provided
+        self.ip = ip or self._extract_ip_from_server()
+    
+    def _extract_ip_from_server(self) -> Optional[str]:
+        """Extract IP address from server URL."""
+        try:
+            # Simple extraction assuming http://ip:port format
+            if "://" in self.server:
+                parts = self.server.split("://")[1].split(":")
+                return parts[0]
+            else:
+                parts = self.server.split(":")
+                return parts[0]
+        except Exception:
+            return None
+    
+    @staticmethod
+    def from_string(proxy_str: str) -> "ProxyConfig":
+        """Create a ProxyConfig from a string in the format 'ip:port:username:password'."""
+        parts = proxy_str.split(":")
+        if len(parts) == 4:  # ip:port:username:password
+            ip, port, username, password = parts
+            return ProxyConfig(
+                server=f"http://{ip}:{port}",
+                username=username,
+                password=password,
+                ip=ip
+            )
+        elif len(parts) == 2:  # ip:port only
+            ip, port = parts
+            return ProxyConfig(
+                server=f"http://{ip}:{port}",
+                ip=ip
+            )
+        else:
+            raise ValueError(f"Invalid proxy string format: {proxy_str}")
+    
+    @staticmethod
+    def from_dict(proxy_dict: Dict) -> "ProxyConfig":
+        """Create a ProxyConfig from a dictionary."""
+        return ProxyConfig(
+            server=proxy_dict.get("server"),
+            username=proxy_dict.get("username"),
+            password=proxy_dict.get("password"),
+            ip=proxy_dict.get("ip")
+        )
+    
+    @staticmethod
+    def from_env(env_var: str = "PROXIES") -> List["ProxyConfig"]:
+        """Load proxies from environment variable.
+        
+        Args:
+            env_var: Name of environment variable containing comma-separated proxy strings
+            
+        Returns:
+            List of ProxyConfig objects
+        """
+        proxies = []
+        try:
+            proxy_list = os.getenv(env_var, "").split(",")
+            for proxy in proxy_list:
+                if not proxy:
+                    continue
+                proxies.append(ProxyConfig.from_string(proxy))
+        except Exception as e:
+            print(f"Error loading proxies from environment: {e}")
+        return proxies
+    
+    def to_dict(self) -> Dict:
+        """Convert to dictionary representation."""
+        return {
+            "server": self.server,
+            "username": self.username,
+            "password": self.password,
+            "ip": self.ip
+        }
+    
+    def clone(self, **kwargs) -> "ProxyConfig":
+        """Create a copy of this configuration with updated values.
+
+        Args:
+            **kwargs: Key-value pairs of configuration options to update
+
+        Returns:
+            ProxyConfig: A new instance with the specified updates
+        """
+        config_dict = self.to_dict()
+        config_dict.update(kwargs)
+        return ProxyConfig.from_dict(config_dict)
 
-from crawl4ai.configs import ProxyConfig
 
 class ProxyRotationStrategy(ABC):
     """Base abstract class for proxy rotation strategies"""

deploy/aws/Dockerfile

@@ -0,0 +1,137 @@
+FROM python:3.10-slim
+
+# Set build arguments
+ARG APP_HOME=/app
+ARG GITHUB_REPO=https://github.com/unclecode/crawl4ai.git
+ARG GITHUB_BRANCH=next
+ARG USE_LOCAL=False
+ARG CONFIG_PATH=""
+
+ENV PYTHONFAULTHANDLER=1 \
+    PYTHONHASHSEED=random \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_DEFAULT_TIMEOUT=100 \
+    DEBIAN_FRONTEND=noninteractive \
+    REDIS_HOST=localhost \
+    REDIS_PORT=6379
+
+ARG PYTHON_VERSION=3.10
+ARG INSTALL_TYPE=default
+ARG ENABLE_GPU=false
+ARG TARGETARCH
+
+LABEL maintainer="unclecode"
+LABEL description="🔥🕷️ Crawl4AI: Open-source LLM Friendly Web Crawler & scraper"
+LABEL version="1.0"    
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    curl \
+    wget \
+    gnupg \
+    git \
+    cmake \
+    pkg-config \
+    python3-dev \
+    libjpeg-dev \
+    redis-server \
+    supervisor \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libglib2.0-0 \
+    libnss3 \
+    libnspr4 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    libcups2 \
+    libdrm2 \
+    libdbus-1-3 \
+    libxcb1 \
+    libxkbcommon0 \
+    libx11-6 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxext6 \
+    libxfixes3 \
+    libxrandr2 \
+    libgbm1 \
+    libpango-1.0-0 \
+    libcairo2 \
+    libasound2 \
+    libatspi2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN if [ "$ENABLE_GPU" = "true" ] && [ "$TARGETARCH" = "amd64" ] ; then \
+    apt-get update && apt-get install -y --no-install-recommends \
+    nvidia-cuda-toolkit \
+    && rm -rf /var/lib/apt/lists/* ; \
+else \
+    echo "Skipping NVIDIA CUDA Toolkit installation (unsupported platform or GPU disabled)"; \
+fi
+
+RUN if [ "$TARGETARCH" = "arm64" ]; then \
+    echo "🦾 Installing ARM-specific optimizations"; \
+    apt-get update && apt-get install -y --no-install-recommends \
+    libopenblas-dev \
+    && rm -rf /var/lib/apt/lists/*; \
+elif [ "$TARGETARCH" = "amd64" ]; then \
+    echo "🖥️ Installing AMD64-specific optimizations"; \
+    apt-get update && apt-get install -y --no-install-recommends \
+    libomp-dev \
+    && rm -rf /var/lib/apt/lists/*; \
+else \
+    echo "Skipping platform-specific optimizations (unsupported platform)"; \
+fi
+
+WORKDIR ${APP_HOME}
+
+RUN git clone --branch ${GITHUB_BRANCH} ${GITHUB_REPO} /tmp/crawl4ai
+
+COPY docker/supervisord.conf .
+COPY docker/requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+RUN if [ "$INSTALL_TYPE" = "all" ] ; then \
+        pip install "/tmp/crawl4ai/[all]" && \
+        python -m nltk.downloader punkt stopwords && \
+        python -m crawl4ai.model_loader ; \
+    elif [ "$INSTALL_TYPE" = "torch" ] ; then \
+        pip install "/tmp/crawl4ai/[torch]" ; \
+    elif [ "$INSTALL_TYPE" = "transformer" ] ; then \
+        pip install "/tmp/crawl4ai/[transformer]" && \
+        python -m crawl4ai.model_loader ; \
+    else \
+        pip install "/tmp/crawl4ai" ; \
+    fi
+    
+RUN pip install --no-cache-dir --upgrade pip && \
+    python -c "import crawl4ai; print('✅ crawl4ai is ready to rock!')" && \
+    python -c "from playwright.sync_api import sync_playwright; print('✅ Playwright is feeling dramatic!')"
+    
+RUN playwright install --with-deps chromium
+
+COPY docker/* ${APP_HOME}/
+RUN if [ -n "$CONFIG_PATH" ] && [ -f "$CONFIG_PATH" ]; then \
+    echo "Using custom config from $CONFIG_PATH" && \
+    cp $CONFIG_PATH /app/config.yml; \
+fi
+
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+    CMD bash -c '\
+    MEM=$(free -m | awk "/^Mem:/{print \$2}"); \
+    if [ $MEM -lt 2048 ]; then \
+        echo "⚠️ Warning: Less than 2GB RAM available! Your container might need a memory boost! 🚀"; \
+        exit 1; \
+    fi && \
+    redis-cli ping > /dev/null && \
+    curl -f http://localhost:8000/health || exit 1'
+
+# EXPOSE 6379
+
+CMD ["supervisord", "-c", "supervisord.conf"]
+    

deploy/aws/deploy-config.yml

@@ -0,0 +1,3 @@
+project_name: PROJECT_NAME
+domain_name: DOMAIN_NAME
+aws_region: AWS_REGION

deploy/aws/deploy.py

@@ -0,0 +1,729 @@
+#!/usr/bin/env python3
+import argparse
+import subprocess
+import sys
+import time
+import json
+import yaml
+import requests
+import os
+
+# Steps for deployment
+STEPS = [
+    "refresh_aws_auth",
+    "fetch_or_create_vpc_and_subnets",
+    "create_ecr_repositories",
+    "create_iam_role",
+    "create_security_groups",
+    "request_acm_certificate",
+    "build_and_push_docker",
+    "create_task_definition",
+    "setup_alb",
+    "deploy_ecs_service",
+    "configure_custom_domain",
+    "test_endpoints"
+]
+
+# Utility function to prompt user for confirmation
+def confirm_step(step_name):
+    while True:
+        response = input(f"Proceed with {step_name}? (yes/no): ").strip().lower()
+        if response in ["yes", "no"]:
+            return response == "yes"
+        print("Please enter 'yes' or 'no'.")
+
+# Utility function to run AWS CLI or shell commands and handle errors
+def run_command(command, error_message, additional_diagnostics=None, cwd="."):
+    try:
+        result = subprocess.run(command, capture_output=True, text=True, check=True, cwd=cwd)
+        return result
+    except subprocess.CalledProcessError as e:
+        with open("error_context.md", "w") as f:
+            f.write(f"{error_message}:\n")
+            f.write(f"Command: {' '.join(command)}\n")
+            f.write(f"Exit Code: {e.returncode}\n")
+            f.write(f"Stdout: {e.stdout}\n")
+            f.write(f"Stderr: {e.stderr}\n")
+            if additional_diagnostics:
+                for diag_cmd in additional_diagnostics:
+                    diag_result = subprocess.run(diag_cmd, capture_output=True, text=True)
+                    f.write(f"\nDiagnostic command: {' '.join(diag_cmd)}\n")
+                    f.write(f"Stdout: {diag_result.stdout}\n")
+                    f.write(f"Stderr: {diag_result.stderr}\n")
+        raise Exception(f"{error_message}: {e.stderr}")
+
+# Utility function to load or initialize state
+def load_state(project_name):
+    state_file = f"{project_name}-state.json"
+    if os.path.exists(state_file):
+        with open(state_file, "r") as f:
+            return json.load(f)
+    return {"last_step": -1}
+
+# Utility function to save state
+def save_state(project_name, state):
+    state_file = f"{project_name}-state.json"
+    with open(state_file, "w") as f:
+        json.dump(state, f, indent=4)
+
+# DNS Check Function
+def check_dns_propagation(domain, alb_dns):
+    try:
+        result = subprocess.run(["dig", "+short", domain], capture_output=True, text=True)
+        if alb_dns in result.stdout:
+            return True
+        return False
+    except Exception as e:
+        print(f"Failed to check DNS: {e}")
+        return False
+
+# Step Functions
+def refresh_aws_auth(project_name, state, config):
+    if state["last_step"] >= 0:
+        print("Skipping refresh_aws_auth (already completed)")
+        return
+    if not confirm_step("Refresh AWS authentication"):
+        sys.exit("User aborted.")
+    run_command(
+        ["aws", "sts", "get-caller-identity"],
+        "Failed to verify AWS credentials"
+    )
+    print("AWS authentication verified.")
+    state["last_step"] = 0
+    save_state(project_name, state)
+
+def fetch_or_create_vpc_and_subnets(project_name, state, config):
+    if state["last_step"] >= 1:
+        print("Skipping fetch_or_create_vpc_and_subnets (already completed)")
+        return state["vpc_id"], state["public_subnets"]
+    if not confirm_step("Fetch or Create VPC and Subnets"):
+        sys.exit("User aborted.")
+    
+    # Fetch AWS account ID
+    result = run_command(
+        ["aws", "sts", "get-caller-identity"],
+        "Failed to get AWS account ID"
+    )
+    account_id = json.loads(result.stdout)["Account"]
+    
+    # Fetch default VPC
+    result = run_command(
+        ["aws", "ec2", "describe-vpcs", "--filters", "Name=isDefault,Values=true", "--region", config["aws_region"]],
+        "Failed to describe VPCs"
+    )
+    vpcs = json.loads(result.stdout).get("Vpcs", [])
+    if not vpcs:
+        result = run_command(
+            ["aws", "ec2", "create-vpc", "--cidr-block", "10.0.0.0/16", "--region", config["aws_region"]],
+            "Failed to create VPC"
+        )
+        vpc_id = json.loads(result.stdout)["Vpc"]["VpcId"]
+        run_command(
+            ["aws", "ec2", "modify-vpc-attribute", "--vpc-id", vpc_id, "--enable-dns-hostnames", "--region", config["aws_region"]],
+            "Failed to enable DNS hostnames"
+        )
+    else:
+        vpc_id = vpcs[0]["VpcId"]
+    
+    # Fetch or create subnets
+    result = run_command(
+        ["aws", "ec2", "describe-subnets", "--filters", f"Name=vpc-id,Values={vpc_id}", "--region", config["aws_region"]],
+        "Failed to describe subnets"
+    )
+    subnets = json.loads(result.stdout).get("Subnets", [])
+    if len(subnets) < 2:
+        azs = json.loads(run_command(
+            ["aws", "ec2", "describe-availability-zones", "--region", config["aws_region"]],
+            "Failed to describe availability zones"
+        ).stdout)["AvailabilityZones"][:2]
+        subnet_ids = []
+        for i, az in enumerate(azs):
+            az_name = az["ZoneName"]
+            result = run_command(
+                ["aws", "ec2", "create-subnet", "--vpc-id", vpc_id, "--cidr-block", f"10.0.{i}.0/24", "--availability-zone", az_name, "--region", config["aws_region"]],
+                f"Failed to create subnet in {az_name}"
+            )
+            subnet_id = json.loads(result.stdout)["Subnet"]["SubnetId"]
+            subnet_ids.append(subnet_id)
+            run_command(
+                ["aws", "ec2", "modify-subnet-attribute", "--subnet-id", subnet_id, "--map-public-ip-on-launch", "--region", config["aws_region"]],
+                f"Failed to make subnet {subnet_id} public"
+            )
+    else:
+        subnet_ids = [s["SubnetId"] for s in subnets[:2]]
+    
+    # Ensure internet gateway
+    result = run_command(
+        ["aws", "ec2", "describe-internet-gateways", "--filters", f"Name=attachment.vpc-id,Values={vpc_id}", "--region", config["aws_region"]],
+        "Failed to describe internet gateways"
+    )
+    igws = json.loads(result.stdout).get("InternetGateways", [])
+    if not igws:
+        result = run_command(
+            ["aws", "ec2", "create-internet-gateway", "--region", config["aws_region"]],
+            "Failed to create internet gateway"
+        )
+        igw_id = json.loads(result.stdout)["InternetGateway"]["InternetGatewayId"]
+        run_command(
+            ["aws", "ec2", "attach-internet-gateway", "--vpc-id", vpc_id, "--internet-gateway-id", igw_id, "--region", config["aws_region"]],
+            "Failed to attach internet gateway"
+        )
+    
+    state["vpc_id"] = vpc_id
+    state["public_subnets"] = subnet_ids
+    state["last_step"] = 1
+    save_state(project_name, state)
+    print(f"VPC ID: {vpc_id}, Subnets: {subnet_ids}")
+    return vpc_id, subnet_ids
+
+def create_ecr_repositories(project_name, state, config):
+    if state["last_step"] >= 2:
+        print("Skipping create_ecr_repositories (already completed)")
+        return
+    if not confirm_step("Create ECR Repositories"):
+        sys.exit("User aborted.")
+    
+    account_id = json.loads(run_command(
+        ["aws", "sts", "get-caller-identity"],
+        "Failed to get AWS account ID"
+    ).stdout)["Account"]
+    repos = [project_name, f"{project_name}-nginx"]
+    for repo in repos:
+        result = subprocess.run(
+            ["aws", "ecr", "describe-repositories", "--repository-names", repo, "--region", config["aws_region"]],
+            capture_output=True, text=True
+        )
+        if result.returncode != 0:
+            run_command(
+                ["aws", "ecr", "create-repository", "--repository-name", repo, "--region", config["aws_region"]],
+                f"Failed to create ECR repository {repo}"
+            )
+        print(f"ECR repository {repo} is ready.")
+    state["last_step"] = 2
+    save_state(project_name, state)
+
+def create_iam_role(project_name, state, config):
+    if state["last_step"] >= 3:
+        print("Skipping create_iam_role (already completed)")
+        return
+    if not confirm_step("Create IAM Role"):
+        sys.exit("User aborted.")
+    
+    account_id = json.loads(run_command(
+        ["aws", "sts", "get-caller-identity"],
+        "Failed to get AWS account ID"
+    ).stdout)["Account"]
+    role_name = "ecsTaskExecutionRole"
+    trust_policy = {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Principal": {"Service": "ecs-tasks.amazonaws.com"},
+                "Action": "sts:AssumeRole"
+            }
+        ]
+    }
+    with open("trust_policy.json", "w") as f:
+        json.dump(trust_policy, f)
+    
+    result = subprocess.run(
+        ["aws", "iam", "get-role", "--role-name", role_name],
+        capture_output=True, text=True
+    )
+    if result.returncode != 0:
+        run_command(
+            ["aws", "iam", "create-role", "--role-name", role_name, "--assume-role-policy-document", "file://trust_policy.json"],
+            f"Failed to create IAM role {role_name}"
+        )
+    run_command(
+        ["aws", "iam", "attach-role-policy", "--role-name", role_name, "--policy-arn", "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"],
+        "Failed to attach ECS task execution policy"
+    )
+    os.remove("trust_policy.json")
+    state["execution_role_arn"] = f"arn:aws:iam::{account_id}:role/{role_name}"
+    state["last_step"] = 3
+    save_state(project_name, state)
+    print(f"IAM role {role_name} configured.")
+
+def create_security_groups(project_name, state, config):
+    if state["last_step"] >= 4:
+        print("Skipping create_security_groups (already completed)")
+        return state["alb_sg_id"], state["ecs_sg_id"]
+    if not confirm_step("Create Security Groups"):
+        sys.exit("User aborted.")
+    
+    vpc_id = state["vpc_id"]
+    alb_sg_name = f"{project_name}-alb-sg"
+    result = run_command(
+        ["aws", "ec2", "describe-security-groups", "--filters", f"Name=vpc-id,Values={vpc_id}", f"Name=group-name,Values={alb_sg_name}", "--region", config["aws_region"]],
+        "Failed to describe ALB security group"
+    )
+    if not json.loads(result.stdout).get("SecurityGroups"):
+        result = run_command(
+            ["aws", "ec2", "create-security-group", "--group-name", alb_sg_name, "--description", "Security group for ALB", "--vpc-id", vpc_id, "--region", config["aws_region"]],
+            "Failed to create ALB security group"
+        )
+        alb_sg_id = json.loads(result.stdout)["GroupId"]
+        run_command(
+            ["aws", "ec2", "authorize-security-group-ingress", "--group-id", alb_sg_id, "--protocol", "tcp", "--port", "80", "--cidr", "0.0.0.0/0", "--region", config["aws_region"]],
+            "Failed to authorize HTTP ingress"
+        )
+        run_command(
+            ["aws", "ec2", "authorize-security-group-ingress", "--group-id", alb_sg_id, "--protocol", "tcp", "--port", "443", "--cidr", "0.0.0.0/0", "--region", config["aws_region"]],
+            "Failed to authorize HTTPS ingress"
+        )
+    else:
+        alb_sg_id = json.loads(result.stdout)["SecurityGroups"][0]["GroupId"]
+    
+    ecs_sg_name = f"{project_name}-ecs-sg"
+    result = run_command(
+        ["aws", "ec2", "describe-security-groups", "--filters", f"Name=vpc-id,Values={vpc_id}", f"Name=group-name,Values={ecs_sg_name}", "--region", config["aws_region"]],
+        "Failed to describe ECS security group"
+    )
+    if not json.loads(result.stdout).get("SecurityGroups"):
+        result = run_command(
+            ["aws", "ec2", "create-security-group", "--group-name", ecs_sg_name, "--description", "Security group for ECS tasks", "--vpc-id", vpc_id, "--region", config["aws_region"]],
+            "Failed to create ECS security group"
+        )
+        ecs_sg_id = json.loads(result.stdout)["GroupId"]
+        run_command(
+            ["aws", "ec2", "authorize-security-group-ingress", "--group-id", ecs_sg_id, "--protocol", "tcp", "--port", "80", "--source-group", alb_sg_id, "--region", config["aws_region"]],
+            "Failed to authorize ECS ingress"
+        )
+    else:
+        ecs_sg_id = json.loads(result.stdout)["SecurityGroups"][0]["GroupId"]
+    
+    state["alb_sg_id"] = alb_sg_id
+    state["ecs_sg_id"] = ecs_sg_id
+    state["last_step"] = 4
+    save_state(project_name, state)
+    print("Security groups configured.")
+    return alb_sg_id, ecs_sg_id
+
+def request_acm_certificate(project_name, state, config):
+    if state["last_step"] >= 5:
+        print("Skipping request_acm_certificate (already completed)")
+        return state["cert_arn"]
+    if not confirm_step("Request ACM Certificate"):
+        sys.exit("User aborted.")
+    
+    domain_name = config["domain_name"]
+    result = run_command(
+        ["aws", "acm", "describe-certificates", "--certificate-statuses", "ISSUED", "--region", config["aws_region"]],
+        "Failed to describe certificates"
+    )
+    certificates = json.loads(result.stdout).get("CertificateSummaryList", [])
+    cert_arn = next((c["CertificateArn"] for c in certificates if c["DomainName"] == domain_name), None)
+    
+    if not cert_arn:
+        result = run_command(
+            ["aws", "acm", "request-certificate", "--domain-name", domain_name, "--validation-method", "DNS", "--region", config["aws_region"]],
+            "Failed to request ACM certificate"
+        )
+        cert_arn = json.loads(result.stdout)["CertificateArn"]
+        
+        time.sleep(10)
+        result = run_command(
+            ["aws", "acm", "describe-certificate", "--certificate-arn", cert_arn, "--region", config["aws_region"]],
+            "Failed to describe certificate"
+        )
+        cert_details = json.loads(result.stdout)["Certificate"]
+        dns_validations = cert_details.get("DomainValidationOptions", [])
+        for validation in dns_validations:
+            if validation["ValidationMethod"] == "DNS" and "ResourceRecord" in validation:
+                record = validation["ResourceRecord"]
+                print(f"Please add this DNS record to validate the certificate for {domain_name}:")
+                print(f"Name: {record['Name']}")
+                print(f"Type: {record['Type']}")
+                print(f"Value: {record['Value']}")
+        print("Press Enter after adding the DNS record...")
+        input()
+        
+        while True:
+            result = run_command(
+                ["aws", "acm", "describe-certificate", "--certificate-arn", cert_arn, "--region", config["aws_region"]],
+                "Failed to check certificate status"
+            )
+            status = json.loads(result.stdout)["Certificate"]["Status"]
+            if status == "ISSUED":
+                break
+            elif status in ["FAILED", "REVOKED", "INACTIVE"]:
+                print("Certificate issuance failed.")
+                sys.exit(1)
+            time.sleep(10)
+    
+    state["cert_arn"] = cert_arn
+    state["last_step"] = 5
+    save_state(project_name, state)
+    print(f"Certificate ARN: {cert_arn}")
+    return cert_arn
+
+def build_and_push_docker(project_name, state, config):
+    if state["last_step"] >= 6:
+        print("Skipping build_and_push_docker (already completed)")
+        return state["fastapi_image"], state["nginx_image"]
+    if not confirm_step("Build and Push Docker Images"):
+        sys.exit("User aborted.")
+    
+    with open("./version.txt", "r") as f:
+        version = f.read().strip()
+    
+    account_id = json.loads(run_command(
+        ["aws", "sts", "get-caller-identity"],
+        "Failed to get AWS account ID"
+    ).stdout)["Account"]
+    region = config["aws_region"]
+    
+    login_password = run_command(
+        ["aws", "ecr", "get-login-password", "--region", region],
+        "Failed to get ECR login password"
+    ).stdout.strip()
+    run_command(
+        ["docker", "login", "--username", "AWS", "--password", login_password, f"{account_id}.dkr.ecr.{region}.amazonaws.com"],
+        "Failed to authenticate Docker to ECR"
+    )
+    
+    fastapi_image = f"{account_id}.dkr.ecr.{region}.amazonaws.com/{project_name}:{version}"
+    run_command(
+        ["docker", "build", "-f", "Dockerfile", "-t", fastapi_image, "."],
+        "Failed to build FastAPI Docker image"
+    )
+    run_command(
+        ["docker", "push", fastapi_image],
+        "Failed to push FastAPI image"
+    )
+    
+    nginx_image = f"{account_id}.dkr.ecr.{region}.amazonaws.com/{project_name}-nginx:{version}"
+    run_command(
+        ["docker", "build", "-f", "Dockerfile", "-t", nginx_image, "."],
+        "Failed to build Nginx Docker image",
+        cwd="./nginx"
+    )
+    run_command(
+        ["docker", "push", nginx_image],
+        "Failed to push Nginx image"
+    )
+    
+    state["fastapi_image"] = fastapi_image
+    state["nginx_image"] = nginx_image
+    state["last_step"] = 6
+    save_state(project_name, state)
+    print("Docker images built and pushed.")
+    return fastapi_image, nginx_image
+
+def create_task_definition(project_name, state, config):
+    if state["last_step"] >= 7:
+        print("Skipping create_task_definition (already completed)")
+        return state["task_def_arn"]
+    if not confirm_step("Create Task Definition"):
+        sys.exit("User aborted.")
+    
+    log_group = f"/ecs/{project_name}-logs"
+    result = run_command(
+        ["aws", "logs", "describe-log-groups", "--log-group-name-prefix", log_group, "--region", config["aws_region"]],
+        "Failed to describe log groups"
+    )
+    if not any(lg["logGroupName"] == log_group for lg in json.loads(result.stdout).get("logGroups", [])):
+        run_command(
+            ["aws", "logs", "create-log-group", "--log-group-name", log_group, "--region", config["aws_region"]],
+            f"Failed to create log group {log_group}"
+        )
+    
+    task_definition = {
+        "family": f"{project_name}-taskdef",
+        "networkMode": "awsvpc",
+        "requiresCompatibilities": ["FARGATE"],
+        "cpu": "512",
+        "memory": "2048",
+        "executionRoleArn": state["execution_role_arn"],
+        "containerDefinitions": [
+            {
+                "name": "fastapi",
+                "image": state["fastapi_image"],
+                "portMappings": [{"containerPort": 8000, "hostPort": 8000, "protocol": "tcp"}],
+                "logConfiguration": {
+                    "logDriver": "awslogs",
+                    "options": {
+                        "awslogs-group": log_group,
+                        "awslogs-region": config["aws_region"],
+                        "awslogs-stream-prefix": "fastapi"
+                    }
+                }
+            },
+            {
+                "name": "nginx",
+                "image": state["nginx_image"],
+                "portMappings": [{"containerPort": 80, "hostPort": 80, "protocol": "tcp"}],
+                "logConfiguration": {
+                    "logDriver": "awslogs",
+                    "options": {
+                        "awslogs-group": log_group,
+                        "awslogs-region": config["aws_region"],
+                        "awslogs-stream-prefix": "nginx"
+                    }
+                }
+            }
+        ]
+    }
+    
+    with open("task_def.json", "w") as f:
+        json.dump(task_definition, f)
+    result = run_command(
+        ["aws", "ecs", "register-task-definition", "--cli-input-json", "file://task_def.json", "--region", config["aws_region"]],
+        "Failed to register task definition"
+    )
+    task_def_arn = json.loads(result.stdout)["taskDefinition"]["taskDefinitionArn"]
+    os.remove("task_def.json")
+    
+    state["task_def_arn"] = task_def_arn
+    state["last_step"] = 7
+    save_state(project_name, state)
+    print("Task definition created.")
+    return task_def_arn
+
+def setup_alb(project_name, state, config):
+    if state["last_step"] >= 8:
+        print("Skipping setup_alb (already completed)")
+        return state["alb_arn"], state["tg_arn"], state["alb_dns"]
+    if not confirm_step("Set Up ALB"):
+        sys.exit("User aborted.")
+    
+    vpc_id = state["vpc_id"]
+    public_subnets = state["public_subnets"]
+    alb_name = f"{project_name}-alb"
+    
+    result = subprocess.run(
+        ["aws", "elbv2", "describe-load-balancers", "--names", alb_name, "--region", config["aws_region"]],
+        capture_output=True, text=True
+    )
+    if result.returncode != 0:
+        run_command(
+            ["aws", "elbv2", "create-load-balancer", "--name", alb_name, "--subnets"] + public_subnets + ["--security-groups", state["alb_sg_id"], "--region", config["aws_region"]],
+            "Failed to create ALB"
+        )
+    alb_arn = json.loads(run_command(
+        ["aws", "elbv2", "describe-load-balancers", "--names", alb_name, "--region", config["aws_region"]],
+        "Failed to describe ALB"
+    ).stdout)["LoadBalancers"][0]["LoadBalancerArn"]
+    alb_dns = json.loads(run_command(
+        ["aws", "elbv2", "describe-load-balancers", "--names", alb_name, "--region", config["aws_region"]],
+        "Failed to get ALB DNS name"
+    ).stdout)["LoadBalancers"][0]["DNSName"]
+    
+    tg_name = f"{project_name}-tg"
+    result = subprocess.run(
+        ["aws", "elbv2", "describe-target-groups", "--names", tg_name, "--region", config["aws_region"]],
+        capture_output=True, text=True
+    )
+    if result.returncode != 0:
+        run_command(
+            ["aws", "elbv2", "create-target-group", "--name", tg_name, "--protocol", "HTTP", "--port", "80", "--vpc-id", vpc_id, "--region", config["aws_region"]],
+            "Failed to create target group"
+        )
+    tg_arn = json.loads(run_command(
+        ["aws", "elbv2", "describe-target-groups", "--names", tg_name, "--region", config["aws_region"]],
+        "Failed to describe target group"
+    ).stdout)["TargetGroups"][0]["TargetGroupArn"]
+    
+    result = run_command(
+        ["aws", "elbv2", "describe-listeners", "--load-balancer-arn", alb_arn, "--region", config["aws_region"]],
+        "Failed to describe listeners"
+    )
+    listeners = json.loads(result.stdout).get("Listeners", [])
+    if not any(l["Port"] == 80 for l in listeners):
+        run_command(
+            ["aws", "elbv2", "create-listener", "--load-balancer-arn", alb_arn, "--protocol", "HTTP", "--port", "80", "--default-actions", "Type=redirect,RedirectConfig={Protocol=HTTPS,Port=443,StatusCode=HTTP_301}", "--region", config["aws_region"]],
+            "Failed to create HTTP listener"
+        )
+    if not any(l["Port"] == 443 for l in listeners):
+        run_command(
+            ["aws", "elbv2", "create-listener", "--load-balancer-arn", alb_arn, "--protocol", "HTTPS", "--port", "443", "--certificates", f"CertificateArn={state['cert_arn']}", "--default-actions", f"Type=forward,TargetGroupArn={tg_arn}", "--region", config["aws_region"]],
+            "Failed to create HTTPS listener"
+        )
+    
+    state["alb_arn"] = alb_arn
+    state["tg_arn"] = tg_arn
+    state["alb_dns"] = alb_dns
+    state["last_step"] = 8
+    save_state(project_name, state)
+    print("ALB configured.")
+    return alb_arn, tg_arn, alb_dns
+
+def deploy_ecs_service(project_name, state, config):
+    if state["last_step"] >= 9:
+        print("Skipping deploy_ecs_service (already completed)")
+        return
+    if not confirm_step("Deploy ECS Service"):
+        sys.exit("User aborted.")
+    
+    cluster_name = f"{project_name}-cluster"
+    result = run_command(
+        ["aws", "ecs", "describe-clusters", "--clusters", cluster_name, "--region", config["aws_region"]],
+        "Failed to describe clusters"
+    )
+    if not json.loads(result.stdout).get("clusters"):
+        run_command(
+            ["aws", "ecs", "create-cluster", "--cluster-name", cluster_name, "--region", config["aws_region"]],
+            "Failed to create ECS cluster"
+        )
+    
+    service_name = f"{project_name}-service"
+    result = run_command(
+        ["aws", "ecs", "describe-services", "--cluster", cluster_name, "--services", service_name, "--region", config["aws_region"]],
+        "Failed to describe services",
+        additional_diagnostics=[["aws", "ecs", "list-tasks", "--cluster", cluster_name, "--service-name", service_name, "--region", config["aws_region"]]]
+    )
+    services = json.loads(result.stdout).get("services", [])
+    if not services or services[0]["status"] == "INACTIVE":
+        run_command(
+            ["aws", "ecs", "create-service", "--cluster", cluster_name, "--service-name", service_name, "--task-definition", state["task_def_arn"], "--desired-count", "1", "--launch-type", "FARGATE", "--network-configuration", f"awsvpcConfiguration={{subnets={json.dumps(state['public_subnets'])},securityGroups=[{state['ecs_sg_id']}],assignPublicIp=ENABLED}}", "--load-balancers", f"targetGroupArn={state['tg_arn']},containerName=nginx,containerPort=80", "--region", config["aws_region"]],
+            "Failed to create ECS service"
+        )
+    else:
+        run_command(
+            ["aws", "ecs", "update-service", "--cluster", cluster_name, "--service", service_name, "--task-definition", state["task_def_arn"], "--region", config["aws_region"]],
+            "Failed to update ECS service"
+        )
+    
+    state["last_step"] = 9
+    save_state(project_name, state)
+    print("ECS service deployed.")
+
+def configure_custom_domain(project_name, state, config):
+    if state["last_step"] >= 10:
+        print("Skipping configure_custom_domain (already completed)")
+        return
+    if not confirm_step("Configure Custom Domain"):
+        sys.exit("User aborted.")
+    
+    domain_name = config["domain_name"]
+    alb_dns = state["alb_dns"]
+    print(f"Please add a CNAME record for {domain_name} pointing to {alb_dns} in your DNS provider.")
+    print("Press Enter after updating the DNS record...")
+    input()
+    
+    while not check_dns_propagation(domain_name, alb_dns):
+        print("DNS propagation not complete. Waiting 30 seconds before retrying...")
+        time.sleep(30)
+    print("DNS propagation confirmed.")
+    
+    state["last_step"] = 10
+    save_state(project_name, state)
+    print("Custom domain configured.")
+
+def test_endpoints(project_name, state, config):
+    if state["last_step"] >= 11:
+        print("Skipping test_endpoints (already completed)")
+        return
+    if not confirm_step("Test Endpoints"):
+        sys.exit("User aborted.")
+    
+    domain = config["domain_name"]
+    time.sleep(30)  # Wait for service to stabilize
+    
+    response = requests.get(f"https://{domain}/health", verify=False)
+    if response.status_code != 200:
+        with open("error_context.md", "w") as f:
+            f.write("Health endpoint test failed:\n")
+            f.write(f"Status Code: {response.status_code}\n")
+            f.write(f"Response: {response.text}\n")
+        sys.exit(1)
+    print("Health endpoint test passed.")
+    
+    payload = {
+        "urls": ["https://example.com"],
+        "browser_config": {"headless": True},
+        "crawler_config": {"stream": False}
+    }
+    response = requests.post(f"https://{domain}/crawl", json=payload, verify=False)
+    if response.status_code != 200:
+        with open("error_context.md", "w") as f:
+            f.write("Crawl endpoint test failed:\n")
+            f.write(f"Status Code: {response.status_code}\n")
+            f.write(f"Response: {response.text}\n")
+        sys.exit(1)
+    print("Crawl endpoint test passed.")
+    
+    state["last_step"] = 11
+    save_state(project_name, state)
+    print("Endpoints tested successfully.")
+
+# Main Deployment Function
+def deploy(project_name, force=False):
+    config_file = f"{project_name}-config.yml"
+    if not os.path.exists(config_file):
+        print(f"Configuration file {config_file} not found. Run 'init' first.")
+        sys.exit(1)
+    
+    with open(config_file, "r") as f:
+        config = yaml.safe_load(f)
+    
+    state = load_state(project_name)
+    if force:
+        state = {"last_step": -1}
+    
+    last_step = state.get("last_step", -1)
+    
+    for step_idx, step_name in enumerate(STEPS):
+        if step_idx <= last_step:
+            print(f"Skipping {step_name} (already completed)")
+            continue
+        print(f"Executing step: {step_name}")
+        func = globals()[step_name]
+        if step_name == "fetch_or_create_vpc_and_subnets":
+            vpc_id, public_subnets = func(project_name, state, config)
+        elif step_name == "create_security_groups":
+            alb_sg_id, ecs_sg_id = func(project_name, state, config)
+        elif step_name == "request_acm_certificate":
+            cert_arn = func(project_name, state, config)
+        elif step_name == "build_and_push_docker":
+            fastapi_image, nginx_image = func(project_name, state, config)
+        elif step_name == "create_task_definition":
+            task_def_arn = func(project_name, state, config)
+        elif step_name == "setup_alb":
+            alb_arn, tg_arn, alb_dns = func(project_name, state, config)
+        elif step_name == "deploy_ecs_service":
+            func(project_name, state, config)
+        elif step_name == "configure_custom_domain":
+            func(project_name, state, config)
+        elif step_name == "test_endpoints":
+            func(project_name, state, config)
+        else:
+            func(project_name, state, config)
+
+# Init Command
+def init(project_name, domain_name, aws_region):
+    config = {
+        "project_name": project_name,
+        "domain_name": domain_name,
+        "aws_region": aws_region
+    }
+    config_file = f"{project_name}-config.yml"
+    with open(config_file, "w") as f:
+        yaml.dump(config, f)
+    print(f"Configuration file {config_file} created.")
+
+# Argument Parser
+parser = argparse.ArgumentParser(description="Crawl4AI Deployment Script")
+subparsers = parser.add_subparsers(dest="command")
+
+# Init Parser
+init_parser = subparsers.add_parser("init", help="Initialize configuration")
+init_parser.add_argument("--project", required=True, help="Project name")
+init_parser.add_argument("--domain", required=True, help="Domain name")
+init_parser.add_argument("--region", required=True, help="AWS region")
+
+# Deploy Parser
+deploy_parser = subparsers.add_parser("deploy", help="Deploy the project")
+deploy_parser.add_argument("--project", required=True, help="Project name")
+deploy_parser.add_argument("--force", action="store_true", help="Force redeployment from start")
+
+args = parser.parse_args()
+
+if args.command == "init":
+    init(args.project, args.domain, args.region)
+elif args.command == "deploy":
+    deploy(args.project, args.force)
+else:
+    parser.print_help()

deploy/aws/docker/.dockerignore

@@ -0,0 +1,31 @@
+# .dockerignore
+*
+
+# Allow specific files and directories when using local installation
+!crawl4ai/
+!docs/
+!deploy/docker/
+!setup.py
+!pyproject.toml
+!README.md
+!LICENSE
+!MANIFEST.in
+!setup.cfg
+!mkdocs.yml
+
+.git/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.DS_Store
+.env
+.venv
+venv/
+tests/
+coverage.xml
+*.log
+*.swp
+*.egg-info/
+dist/
+build/

deploy/aws/docker/.llm.env.example

@@ -0,0 +1,8 @@
+# LLM Provider Keys
+OPENAI_API_KEY=your_openai_key_here
+DEEPSEEK_API_KEY=your_deepseek_key_here
+ANTHROPIC_API_KEY=your_anthropic_key_here
+GROQ_API_KEY=your_groq_key_here
+TOGETHER_API_KEY=your_together_key_here
+MISTRAL_API_KEY=your_mistral_key_here
+GEMINI_API_TOKEN=your_gemini_key_here

deploy/aws/docker/README.md

@@ -0,0 +1,847 @@
+# Crawl4AI Docker Guide 🐳
+
+## Table of Contents
+- [Prerequisites](#prerequisites)
+- [Installation](#installation)
+  - [Local Build](#local-build)
+  - [Docker Hub](#docker-hub)
+- [Dockerfile Parameters](#dockerfile-parameters)
+- [Using the API](#using-the-api)
+  - [Understanding Request Schema](#understanding-request-schema)
+  - [REST API Examples](#rest-api-examples)
+  - [Python SDK](#python-sdk)
+- [Metrics & Monitoring](#metrics--monitoring)
+- [Deployment Scenarios](#deployment-scenarios)
+- [Complete Examples](#complete-examples)
+- [Getting Help](#getting-help)
+
+## Prerequisites
+
+Before we dive in, make sure you have:
+- Docker installed and running (version 20.10.0 or higher)
+- At least 4GB of RAM available for the container
+- Python 3.10+ (if using the Python SDK)
+- Node.js 16+ (if using the Node.js examples)
+
+> 💡 **Pro tip**: Run `docker info` to check your Docker installation and available resources.
+
+## Installation
+
+### Local Build
+
+Let's get your local environment set up step by step!
+
+#### 1. Building the Image
+
+First, clone the repository and build the Docker image:
+
+```bash
+# Clone the repository
+git clone https://github.com/unclecode/crawl4ai.git
+cd crawl4ai/deploy
+
+# Build the Docker image
+docker build --platform=linux/amd64 --no-cache -t crawl4ai .
+
+# Or build for arm64
+docker build --platform=linux/arm64 --no-cache -t crawl4ai .
+```
+
+#### 2. Environment Setup
+
+If you plan to use LLMs (Language Models), you'll need to set up your API keys. Create a `.llm.env` file:
+
+```env
+# OpenAI
+OPENAI_API_KEY=sk-your-key
+
+# Anthropic
+ANTHROPIC_API_KEY=your-anthropic-key
+
+# DeepSeek
+DEEPSEEK_API_KEY=your-deepseek-key
+
+# Check out https://docs.litellm.ai/docs/providers for more providers!
+```
+
+> 🔑 **Note**: Keep your API keys secure! Never commit them to version control.
+
+#### 3. Running the Container
+
+You have several options for running the container:
+
+Basic run (no LLM support):
+```bash
+docker run -d -p 8000:8000 --name crawl4ai crawl4ai
+```
+
+With LLM support:
+```bash
+docker run -d -p 8000:8000 \
+  --env-file .llm.env \
+  --name crawl4ai \
+  crawl4ai
+```
+
+Using host environment variables (Not a good practice, but works for local testing):
+```bash
+docker run -d -p 8000:8000 \
+  --env-file .llm.env \
+  --env "$(env)" \
+  --name crawl4ai \
+  crawl4ai
+```
+
+#### Multi-Platform Build
+For distributing your image across different architectures, use `buildx`:
+
+```bash
+# Set up buildx builder
+docker buildx create --use
+
+# Build for multiple platforms
+docker buildx build \
+  --platform linux/amd64,linux/arm64 \
+  -t crawl4ai \
+  --push \
+  .
+```
+
+> 💡 **Note**: Multi-platform builds require Docker Buildx and need to be pushed to a registry.
+
+#### Development Build
+For development, you might want to enable all features:
+
+```bash
+docker build -t crawl4ai
+  --build-arg INSTALL_TYPE=all \
+  --build-arg PYTHON_VERSION=3.10 \
+  --build-arg ENABLE_GPU=true \
+  .
+```
+
+#### GPU-Enabled Build
+If you plan to use GPU acceleration:
+
+```bash
+docker build -t crawl4ai
+  --build-arg ENABLE_GPU=true \
+  deploy/docker/
+```
+
+### Build Arguments Explained
+
+| Argument | Description | Default | Options |
+|----------|-------------|---------|----------|
+| PYTHON_VERSION | Python version | 3.10 | 3.8, 3.9, 3.10 |
+| INSTALL_TYPE | Feature set | default | default, all, torch, transformer |
+| ENABLE_GPU | GPU support | false | true, false |
+| APP_HOME | Install path | /app | any valid path |
+
+### Build Best Practices
+
+1. **Choose the Right Install Type**
+   - `default`: Basic installation, smallest image, to be honest, I use this most of the time.
+   - `all`: Full features, larger image (include transformer, and nltk, make sure you really need them)
+
+2. **Platform Considerations**
+   - Let Docker auto-detect platform unless you need cross-compilation
+   - Use --platform for specific architecture requirements
+   - Consider buildx for multi-architecture distribution
+
+3. **Performance Optimization**
+   - The image automatically includes platform-specific optimizations
+   - AMD64 gets OpenMP optimizations
+   - ARM64 gets OpenBLAS optimizations
+
+### Docker Hub
+
+> 🚧 Coming soon! The image will be available at `crawl4ai`. Stay tuned!
+
+## Using the API
+
+In the following sections, we discuss two ways to communicate with the Docker server. One option is to use the client SDK that I developed for Python, and I will soon develop one for Node.js. I highly recommend this approach to avoid mistakes. Alternatively, you can take a more technical route by using the JSON structure and passing it to all the URLs, which I will explain in detail.
+
+### Python SDK
+
+The SDK makes things easier! Here's how to use it:
+
+```python
+from crawl4ai.docker_client import Crawl4aiDockerClient
+from crawl4ai import BrowserConfig, CrawlerRunConfig
+
+async def main():
+    async with Crawl4aiDockerClient(base_url="http://localhost:8000", verbose=True) as client:
+      # If JWT is enabled, you can authenticate like this: (more on this later)
+        # await client.authenticate("test@example.com")
+        
+        # Non-streaming crawl
+        results = await client.crawl(
+            ["https://example.com", "https://python.org"],
+            browser_config=BrowserConfig(headless=True),
+            crawler_config=CrawlerRunConfig()
+        )
+        print(f"Non-streaming results: {results}")
+        
+        # Streaming crawl
+        crawler_config = CrawlerRunConfig(stream=True)
+        async for result in await client.crawl(
+            ["https://example.com", "https://python.org"],
+            browser_config=BrowserConfig(headless=True),
+            crawler_config=crawler_config
+        ):
+            print(f"Streamed result: {result}")
+        
+        # Get schema
+        schema = await client.get_schema()
+        print(f"Schema: {schema}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+`Crawl4aiDockerClient` is an async context manager that handles the connection for you. You can pass in optional parameters for more control:
+
+- `base_url` (str): Base URL of the Crawl4AI Docker server
+- `timeout` (float): Default timeout for requests in seconds
+- `verify_ssl` (bool): Whether to verify SSL certificates
+- `verbose` (bool): Whether to show logging output
+- `log_file` (str, optional): Path to log file if file logging is desired
+
+This client SDK generates a properly structured JSON request for the server's HTTP API.
+
+## Second Approach: Direct API Calls
+
+This is super important! The API expects a specific structure that matches our Python classes. Let me show you how it works.
+
+### Understanding Configuration Structure
+
+Let's dive deep into how configurations work in Crawl4AI. Every configuration object follows a consistent pattern of `type` and `params`. This structure enables complex, nested configurations while maintaining clarity.
+
+#### The Basic Pattern
+
+Try this in Python to understand the structure:
+```python
+from crawl4ai import BrowserConfig
+
+# Create a config and see its structure
+config = BrowserConfig(headless=True)
+print(config.dump())
+```
+
+This outputs:
+```json
+{
+    "type": "BrowserConfig",
+    "params": {
+        "headless": true
+    }
+}
+```
+
+#### Simple vs Complex Values
+
+The structure follows these rules:
+- Simple values (strings, numbers, booleans, lists) are passed directly
+- Complex values (classes, dictionaries) use the type-params pattern
+
+For example, with dictionaries:
+```json
+{
+    "browser_config": {
+        "type": "BrowserConfig",
+        "params": {
+            "headless": true,           // Simple boolean - direct value
+            "viewport": {               // Complex dictionary - needs type-params
+                "type": "dict",
+                "value": {
+                    "width": 1200,
+                    "height": 800
+                }
+            }
+        }
+    }
+}
+```
+
+#### Strategy Pattern and Nesting
+
+Strategies (like chunking or content filtering) demonstrate why we need this structure. Consider this chunking configuration:
+
+```json
+{
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "chunking_strategy": {
+                "type": "RegexChunking",      // Strategy implementation
+                "params": {
+                    "patterns": ["\n\n", "\\.\\s+"]
+                }
+            }
+        }
+    }
+}
+```
+
+Here, `chunking_strategy` accepts any chunking implementation. The `type` field tells the system which strategy to use, and `params` configures that specific strategy.
+
+#### Complex Nested Example
+
+Let's look at a more complex example with content filtering:
+
+```json
+{
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "markdown_generator": {
+                "type": "DefaultMarkdownGenerator",
+                "params": {
+                    "content_filter": {
+                        "type": "PruningContentFilter",
+                        "params": {
+                            "threshold": 0.48,
+                            "threshold_type": "fixed"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+```
+
+This shows how deeply configurations can nest while maintaining a consistent structure.
+
+#### Quick Grammar Overview
+```
+config := {
+    "type": string,
+    "params": {
+        key: simple_value | complex_value
+    }
+}
+
+simple_value := string | number | boolean | [simple_value]
+complex_value := config | dict_value
+
+dict_value := {
+    "type": "dict",
+    "value": object
+}
+```
+
+#### Important Rules 🚨
+
+- Always use the type-params pattern for class instances
+- Use direct values for primitives (numbers, strings, booleans)
+- Wrap dictionaries with {"type": "dict", "value": {...}}
+- Arrays/lists are passed directly without type-params
+- All parameters are optional unless specifically required
+
+#### Pro Tip 💡
+
+The easiest way to get the correct structure is to:
+1. Create configuration objects in Python
+2. Use the `dump()` method to see their JSON representation
+3. Use that JSON in your API calls
+
+Example:
+```python
+from crawl4ai import CrawlerRunConfig, PruningContentFilter
+
+config = CrawlerRunConfig(
+    content_filter=PruningContentFilter(threshold=0.48)
+)
+print(config.dump())  # Use this JSON in your API calls
+```
+
+
+#### More Examples
+
+**Advanced Crawler Configuration**
+
+```json
+{
+    "urls": ["https://example.com"],
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "cache_mode": "bypass",
+            "markdown_generator": {
+                "type": "DefaultMarkdownGenerator",
+                "params": {
+                    "content_filter": {
+                        "type": "PruningContentFilter",
+                        "params": {
+                            "threshold": 0.48,
+                            "threshold_type": "fixed",
+                            "min_word_threshold": 0
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+```
+
+**Extraction Strategy**:
+
+```json
+{
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "extraction_strategy": {
+                "type": "JsonCssExtractionStrategy",
+                "params": {
+                    "schema": {
+                        "baseSelector": "article.post",
+                        "fields": [
+                            {"name": "title", "selector": "h1", "type": "text"},
+                            {"name": "content", "selector": ".content", "type": "html"}
+                        ]
+                    }
+                }
+            }
+        }
+    }
+}
+```
+
+**LLM Extraction Strategy**
+
+```json
+{
+  "crawler_config": {
+    "type": "CrawlerRunConfig",
+    "params": {
+      "extraction_strategy": {
+        "type": "LLMExtractionStrategy",
+        "params": {
+          "instruction": "Extract article title, author, publication date and main content",
+          "provider": "openai/gpt-4",
+          "api_token": "your-api-token",
+          "schema": {
+            "type": "dict",
+            "value": {
+              "title": "Article Schema",
+              "type": "object",
+              "properties": {
+                "title": {
+                  "type": "string",
+                  "description": "The article's headline"
+                },
+                "author": {
+                  "type": "string",
+                  "description": "The author's name"
+                },
+                "published_date": {
+                  "type": "string",
+                  "format": "date-time",
+                  "description": "Publication date and time"
+                },
+                "content": {
+                  "type": "string",
+                  "description": "The main article content"
+                }
+              },
+              "required": ["title", "content"]
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+**Deep Crawler Example**
+
+```json
+{
+  "crawler_config": {
+    "type": "CrawlerRunConfig",
+    "params": {
+      "deep_crawl_strategy": {
+        "type": "BFSDeepCrawlStrategy",
+        "params": {
+          "max_depth": 3,
+          "max_pages": 100,
+          "filter_chain": {
+            "type": "FastFilterChain",
+            "params": {
+              "filters": [
+                {
+                  "type": "FastContentTypeFilter",
+                  "params": {
+                    "allowed_types": ["text/html", "application/xhtml+xml"]
+                  }
+                },
+                {
+                  "type": "FastDomainFilter",
+                  "params": {
+                    "allowed_domains": ["blog.*", "docs.*"],
+                    "blocked_domains": ["ads.*", "analytics.*"]
+                  }
+                },
+                {
+                  "type": "FastURLPatternFilter",
+                  "params": {
+                    "allowed_patterns": ["^/blog/", "^/docs/"],
+                    "blocked_patterns": [".*/ads/", ".*/sponsored/"]
+                  }
+                }
+              ]
+            }
+          },
+          "url_scorer": {
+            "type": "FastCompositeScorer",
+            "params": {
+              "scorers": [
+                {
+                  "type": "FastKeywordRelevanceScorer",
+                  "params": {
+                    "keywords": ["tutorial", "guide", "documentation"],
+                    "weight": 1.0
+                  }
+                },
+                {
+                  "type": "FastPathDepthScorer",
+                  "params": {
+                    "weight": 0.5,
+                    "preferred_depth": 2
+                  }
+                },
+                {
+                  "type": "FastFreshnessScorer",
+                  "params": {
+                    "weight": 0.8,
+                    "max_age_days": 365
+                  }
+                }
+              ]
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+### REST API Examples
+
+Let's look at some practical examples:
+
+#### Simple Crawl
+
+```python
+import requests
+
+crawl_payload = {
+    "urls": ["https://example.com"],
+    "browser_config": {"headless": True},
+    "crawler_config": {"stream": False}
+}
+response = requests.post(
+    "http://localhost:8000/crawl",
+    # headers={"Authorization": f"Bearer {token}"},  # If JWT is enabled, more on this later
+    json=crawl_payload
+)
+print(response.json())  # Print the response for debugging
+```
+
+#### Streaming Results
+
+```python
+async def test_stream_crawl(session, token: str):
+    """Test the /crawl/stream endpoint with multiple URLs."""
+    url = "http://localhost:8000/crawl/stream"
+    payload = {
+        "urls": [
+            "https://example.com",
+            "https://example.com/page1",  
+            "https://example.com/page2",  
+            "https://example.com/page3",  
+        ],
+        "browser_config": {"headless": True, "viewport": {"width": 1200}},
+        "crawler_config": {"stream": True, "cache_mode": "aggressive"}
+    }
+
+    # headers = {"Authorization": f"Bearer {token}"} # If JWT is enabled, more on this later
+    
+    try:
+        async with session.post(url, json=payload, headers=headers) as response:
+            status = response.status
+            print(f"Status: {status} (Expected: 200)")
+            assert status == 200, f"Expected 200, got {status}"
+            
+            # Read streaming response line-by-line (NDJSON)
+            async for line in response.content:
+                if line:
+                    data = json.loads(line.decode('utf-8').strip())
+                    print(f"Streamed Result: {json.dumps(data, indent=2)}")
+    except Exception as e:
+        print(f"Error in streaming crawl test: {str(e)}")
+```
+
+## Metrics & Monitoring
+
+Keep an eye on your crawler with these endpoints:
+
+- `/health` - Quick health check
+- `/metrics` - Detailed Prometheus metrics
+- `/schema` - Full API schema
+
+Example health check:
+```bash
+curl http://localhost:8000/health
+```
+
+## Deployment Scenarios
+
+> 🚧 Coming soon! We'll cover:
+> - Kubernetes deployment
+> - Cloud provider setups (AWS, GCP, Azure)
+> - High-availability configurations
+> - Load balancing strategies
+
+## Complete Examples
+
+Check out the `examples` folder in our repository for full working examples! Here are two to get you started:
+[Using Client SDK](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_python_sdk_example.py)
+[Using REST API](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_python_rest_api_example.py)
+
+## Server Configuration
+
+The server's behavior can be customized through the `config.yml` file. Let's explore how to configure your Crawl4AI server for optimal performance and security.
+
+### Understanding config.yml
+
+The configuration file is located at `deploy/docker/config.yml`. You can either modify this file before building the image or mount a custom configuration when running the container.
+
+Here's a detailed breakdown of the configuration options:
+
+```yaml
+# Application Configuration
+app:
+  title: "Crawl4AI API"           # Server title in OpenAPI docs
+  version: "1.0.0"               # API version
+  host: "0.0.0.0"               # Listen on all interfaces
+  port: 8000                    # Server port
+  reload: True                  # Enable hot reloading (development only)
+  timeout_keep_alive: 300       # Keep-alive timeout in seconds
+
+# Rate Limiting Configuration
+rate_limiting:
+  enabled: True                 # Enable/disable rate limiting
+  default_limit: "100/minute"   # Rate limit format: "number/timeunit"
+  trusted_proxies: []          # List of trusted proxy IPs
+  storage_uri: "memory://"     # Use "redis://localhost:6379" for production
+
+# Security Configuration
+security:
+  enabled: false               # Master toggle for security features
+  jwt_enabled: true            # Enable JWT authentication
+  https_redirect: True         # Force HTTPS
+  trusted_hosts: ["*"]         # Allowed hosts (use specific domains in production)
+  headers:                     # Security headers
+    x_content_type_options: "nosniff"
+    x_frame_options: "DENY"
+    content_security_policy: "default-src 'self'"
+    strict_transport_security: "max-age=63072000; includeSubDomains"
+
+# Crawler Configuration
+crawler:
+  memory_threshold_percent: 95.0  # Memory usage threshold
+  rate_limiter:
+    base_delay: [1.0, 2.0]      # Min and max delay between requests
+  timeouts:
+    stream_init: 30.0           # Stream initialization timeout
+    batch_process: 300.0        # Batch processing timeout
+
+# Logging Configuration
+logging:
+  level: "INFO"                 # Log level (DEBUG, INFO, WARNING, ERROR)
+  format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+# Observability Configuration
+observability:
+  prometheus:
+    enabled: True              # Enable Prometheus metrics
+    endpoint: "/metrics"       # Metrics endpoint
+  health_check:
+    endpoint: "/health"        # Health check endpoint
+```
+
+### JWT Authentication
+
+When `security.jwt_enabled` is set to `true` in your config.yml, all endpoints require JWT authentication via bearer tokens. Here's how it works:
+
+#### Getting a Token
+```python
+POST /token
+Content-Type: application/json
+
+{
+    "email": "user@example.com"
+}
+```
+
+The endpoint returns:
+```json
+{
+    "email": "user@example.com",
+    "access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOi...",
+    "token_type": "bearer"
+}
+```
+
+#### Using the Token
+Add the token to your requests:
+```bash
+curl -H "Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGci..." http://localhost:8000/crawl
+```
+
+Using the Python SDK:
+```python
+from crawl4ai.docker_client import Crawl4aiDockerClient
+
+async with Crawl4aiDockerClient() as client:
+    # Authenticate first
+    await client.authenticate("user@example.com")
+    
+    # Now all requests will include the token automatically
+    result = await client.crawl(urls=["https://example.com"])
+```
+
+#### Production Considerations 💡
+The default implementation uses a simple email verification. For production use, consider:
+- Email verification via OTP/magic links
+- OAuth2 integration
+- Rate limiting token generation
+- Token expiration and refresh mechanisms
+- IP-based restrictions
+
+### Configuration Tips and Best Practices
+
+1. **Production Settings** 🏭
+
+   ```yaml
+   app:
+     reload: False              # Disable reload in production
+     timeout_keep_alive: 120    # Lower timeout for better resource management
+   
+   rate_limiting:
+     storage_uri: "redis://redis:6379"  # Use Redis for distributed rate limiting
+     default_limit: "50/minute"         # More conservative rate limit
+   
+   security:
+     enabled: true                      # Enable all security features
+     trusted_hosts: ["your-domain.com"] # Restrict to your domain
+   ```
+
+2. **Development Settings** 🛠️
+
+   ```yaml
+   app:
+     reload: True               # Enable hot reloading
+     timeout_keep_alive: 300    # Longer timeout for debugging
+   
+   logging:
+     level: "DEBUG"            # More verbose logging
+   ```
+
+3. **High-Traffic Settings** 🚦
+
+   ```yaml
+   crawler:
+     memory_threshold_percent: 85.0  # More conservative memory limit
+     rate_limiter:
+       base_delay: [2.0, 4.0]       # More aggressive rate limiting
+   ```
+
+### Customizing Your Configuration
+
+#### Method 1: Pre-build Configuration
+
+```bash
+# Copy and modify config before building
+cd crawl4ai/deploy
+vim custom-config.yml # Or use any editor
+
+# Build with custom config
+docker build --platform=linux/amd64 --no-cache -t crawl4ai:latest .
+```
+
+#### Method 2: Build-time Configuration
+
+Use a custom config during build:
+
+```bash
+# Build with custom config
+docker build --platform=linux/amd64 --no-cache \
+  --build-arg CONFIG_PATH=/path/to/custom-config.yml \ 
+  -t crawl4ai:latest .
+```
+
+#### Method 3: Runtime Configuration
+```bash
+# Mount custom config at runtime
+docker run -d -p 8000:8000 \
+  -v $(pwd)/custom-config.yml:/app/config.yml \
+  crawl4ai-server:prod
+```
+
+> 💡 Note: When using Method 2, `/path/to/custom-config.yml` is relative to deploy directory.
+> 💡 Note: When using Method 3, ensure your custom config file has all required fields as the container will use this instead of the built-in config.
+
+### Configuration Recommendations
+
+1. **Security First** 🔒
+   - Always enable security in production
+   - Use specific trusted_hosts instead of wildcards
+   - Set up proper rate limiting to protect your server
+   - Consider your environment before enabling HTTPS redirect
+
+2. **Resource Management** 💻
+   - Adjust memory_threshold_percent based on available RAM
+   - Set timeouts according to your content size and network conditions
+   - Use Redis for rate limiting in multi-container setups
+
+3. **Monitoring** 📊
+   - Enable Prometheus if you need metrics
+   - Set DEBUG logging in development, INFO in production
+   - Regular health check monitoring is crucial
+
+4. **Performance Tuning** ⚡
+   - Start with conservative rate limiter delays
+   - Increase batch_process timeout for large content
+   - Adjust stream_init timeout based on initial response times
+
+## Getting Help
+
+We're here to help you succeed with Crawl4AI! Here's how to get support:
+
+- 📖 Check our [full documentation](https://docs.crawl4ai.com)
+- 🐛 Found a bug? [Open an issue](https://github.com/unclecode/crawl4ai/issues)
+- 💬 Join our [Discord community](https://discord.gg/crawl4ai)
+- ⭐ Star us on GitHub to show support!
+
+## Summary
+
+In this guide, we've covered everything you need to get started with Crawl4AI's Docker deployment:
+- Building and running the Docker container
+- Configuring the environment
+- Making API requests with proper typing
+- Using the Python SDK
+- Monitoring your deployment
+
+Remember, the examples in the `examples` folder are your friends - they show real-world usage patterns that you can adapt for your needs.
+
+Keep exploring, and don't hesitate to reach out if you need help! We're building something amazing together. 🚀
+
+Happy crawling! 🕷️

deploy/aws/docker/api.py

@@ -0,0 +1,442 @@
+import os
+import json
+import asyncio
+from typing import List, Tuple
+
+import logging
+from typing import Optional, AsyncGenerator
+from urllib.parse import unquote
+from fastapi import HTTPException, Request, status
+from fastapi.background import BackgroundTasks
+from fastapi.responses import JSONResponse
+from redis import asyncio as aioredis
+
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    LLMExtractionStrategy,
+    CacheMode,
+    BrowserConfig,
+    MemoryAdaptiveDispatcher,
+    RateLimiter
+)
+from crawl4ai.utils import perform_completion_with_backoff
+from crawl4ai.content_filter_strategy import (
+    PruningContentFilter,
+    BM25ContentFilter,
+    LLMContentFilter
+)
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
+
+from utils import (
+    TaskStatus,
+    FilterType,
+    get_base_url,
+    is_task_id,
+    should_cleanup_task,
+    decode_redis_hash
+)
+
+logger = logging.getLogger(__name__)
+
+async def handle_llm_qa(
+    url: str,
+    query: str,
+    config: dict
+) -> str:
+    """Process QA using LLM with crawled content as context."""
+    try:
+        # Extract base URL by finding last '?q=' occurrence
+        last_q_index = url.rfind('?q=')
+        if last_q_index != -1:
+            url = url[:last_q_index]
+
+        # Get markdown content
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(url)
+            if not result.success:
+                raise HTTPException(
+                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                    detail=result.error_message
+                )
+            content = result.markdown_v2.fit_markdown
+
+        # Create prompt and get LLM response
+        prompt = f"""Use the following content as context to answer the question.
+    Content:
+    {content}
+
+    Question: {query}
+
+    Answer:"""
+
+        response = perform_completion_with_backoff(
+            provider=config["llm"]["provider"],
+            prompt_with_variables=prompt,
+            api_token=os.environ.get(config["llm"].get("api_key_env", ""))
+        )
+
+        return response.choices[0].message.content
+    except Exception as e:
+        logger.error(f"QA processing error: {str(e)}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=str(e)
+        )
+
+async def process_llm_extraction(
+    redis: aioredis.Redis,
+    config: dict,
+    task_id: str,
+    url: str,
+    instruction: str,
+    schema: Optional[str] = None,
+    cache: str = "0"
+) -> None:
+    """Process LLM extraction in background."""
+    try:
+        # If config['llm'] has api_key then ignore the api_key_env
+        api_key = ""
+        if "api_key" in config["llm"]:
+            api_key = config["llm"]["api_key"]
+        else:
+            api_key = os.environ.get(config["llm"].get("api_key_env", None), "")
+        llm_strategy = LLMExtractionStrategy(
+            provider=config["llm"]["provider"],
+            api_token=api_key,
+            instruction=instruction,
+            schema=json.loads(schema) if schema else None,
+        )
+
+        cache_mode = CacheMode.ENABLED if cache == "1" else CacheMode.WRITE_ONLY
+
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(
+                url=url,
+                config=CrawlerRunConfig(
+                    extraction_strategy=llm_strategy,
+                    scraping_strategy=LXMLWebScrapingStrategy(),
+                    cache_mode=cache_mode
+                )
+            )
+
+        if not result.success:
+            await redis.hset(f"task:{task_id}", mapping={
+                "status": TaskStatus.FAILED,
+                "error": result.error_message
+            })
+            return
+
+        try:
+            content = json.loads(result.extracted_content)
+        except json.JSONDecodeError:
+            content = result.extracted_content
+        await redis.hset(f"task:{task_id}", mapping={
+            "status": TaskStatus.COMPLETED,
+            "result": json.dumps(content)
+        })
+
+    except Exception as e:
+        logger.error(f"LLM extraction error: {str(e)}", exc_info=True)
+        await redis.hset(f"task:{task_id}", mapping={
+            "status": TaskStatus.FAILED,
+            "error": str(e)
+        })
+
+async def handle_markdown_request(
+    url: str,
+    filter_type: FilterType,
+    query: Optional[str] = None,
+    cache: str = "0",
+    config: Optional[dict] = None
+) -> str:
+    """Handle markdown generation requests."""
+    try:
+        decoded_url = unquote(url)
+        if not decoded_url.startswith(('http://', 'https://')):
+            decoded_url = 'https://' + decoded_url
+
+        if filter_type == FilterType.RAW:
+            md_generator = DefaultMarkdownGenerator()
+        else:
+            content_filter = {
+                FilterType.FIT: PruningContentFilter(),
+                FilterType.BM25: BM25ContentFilter(user_query=query or ""),
+                FilterType.LLM: LLMContentFilter(
+                    provider=config["llm"]["provider"],
+                    api_token=os.environ.get(config["llm"].get("api_key_env", None), ""),
+                    instruction=query or "Extract main content"
+                )
+            }[filter_type]
+            md_generator = DefaultMarkdownGenerator(content_filter=content_filter)
+
+        cache_mode = CacheMode.ENABLED if cache == "1" else CacheMode.WRITE_ONLY
+
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(
+                url=decoded_url,
+                config=CrawlerRunConfig(
+                    markdown_generator=md_generator,
+                    scraping_strategy=LXMLWebScrapingStrategy(),
+                    cache_mode=cache_mode
+                )
+            )
+            
+            if not result.success:
+                raise HTTPException(
+                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                    detail=result.error_message
+                )
+
+            return (result.markdown_v2.raw_markdown 
+                   if filter_type == FilterType.RAW 
+                   else result.markdown_v2.fit_markdown)
+
+    except Exception as e:
+        logger.error(f"Markdown error: {str(e)}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=str(e)
+        )
+
+async def handle_llm_request(
+    redis: aioredis.Redis,
+    background_tasks: BackgroundTasks,
+    request: Request,
+    input_path: str,
+    query: Optional[str] = None,
+    schema: Optional[str] = None,
+    cache: str = "0",
+    config: Optional[dict] = None
+) -> JSONResponse:
+    """Handle LLM extraction requests."""
+    base_url = get_base_url(request)
+    
+    try:
+        if is_task_id(input_path):
+            return await handle_task_status(
+                redis, input_path, base_url
+            )
+
+        if not query:
+            return JSONResponse({
+                "message": "Please provide an instruction",
+                "_links": {
+                    "example": {
+                        "href": f"{base_url}/llm/{input_path}?q=Extract+main+content",
+                        "title": "Try this example"
+                    }
+                }
+            })
+
+        return await create_new_task(
+            redis,
+            background_tasks,
+            input_path,
+            query,
+            schema,
+            cache,
+            base_url,
+            config
+        )
+
+    except Exception as e:
+        logger.error(f"LLM endpoint error: {str(e)}", exc_info=True)
+        return JSONResponse({
+            "error": str(e),
+            "_links": {
+                "retry": {"href": str(request.url)}
+            }
+        }, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+async def handle_task_status(
+    redis: aioredis.Redis,
+    task_id: str,
+    base_url: str
+) -> JSONResponse:
+    """Handle task status check requests."""
+    task = await redis.hgetall(f"task:{task_id}")
+    if not task:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Task not found"
+        )
+
+    task = decode_redis_hash(task)
+    response = create_task_response(task, task_id, base_url)
+
+    if task["status"] in [TaskStatus.COMPLETED, TaskStatus.FAILED]:
+        if should_cleanup_task(task["created_at"]):
+            await redis.delete(f"task:{task_id}")
+
+    return JSONResponse(response)
+
+async def create_new_task(
+    redis: aioredis.Redis,
+    background_tasks: BackgroundTasks,
+    input_path: str,
+    query: str,
+    schema: Optional[str],
+    cache: str,
+    base_url: str,
+    config: dict
+) -> JSONResponse:
+    """Create and initialize a new task."""
+    decoded_url = unquote(input_path)
+    if not decoded_url.startswith(('http://', 'https://')):
+        decoded_url = 'https://' + decoded_url
+
+    from datetime import datetime
+    task_id = f"llm_{int(datetime.now().timestamp())}_{id(background_tasks)}"
+    
+    await redis.hset(f"task:{task_id}", mapping={
+        "status": TaskStatus.PROCESSING,
+        "created_at": datetime.now().isoformat(),
+        "url": decoded_url
+    })
+
+    background_tasks.add_task(
+        process_llm_extraction,
+        redis,
+        config,
+        task_id,
+        decoded_url,
+        query,
+        schema,
+        cache
+    )
+
+    return JSONResponse({
+        "task_id": task_id,
+        "status": TaskStatus.PROCESSING,
+        "url": decoded_url,
+        "_links": {
+            "self": {"href": f"{base_url}/llm/{task_id}"},
+            "status": {"href": f"{base_url}/llm/{task_id}"}
+        }
+    })
+
+def create_task_response(task: dict, task_id: str, base_url: str) -> dict:
+    """Create response for task status check."""
+    response = {
+        "task_id": task_id,
+        "status": task["status"],
+        "created_at": task["created_at"],
+        "url": task["url"],
+        "_links": {
+            "self": {"href": f"{base_url}/llm/{task_id}"},
+            "refresh": {"href": f"{base_url}/llm/{task_id}"}
+        }
+    }
+
+    if task["status"] == TaskStatus.COMPLETED:
+        response["result"] = json.loads(task["result"])
+    elif task["status"] == TaskStatus.FAILED:
+        response["error"] = task["error"]
+
+    return response
+
+async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator) -> AsyncGenerator[bytes, None]:
+    """Stream results with heartbeats and completion markers."""
+    import json
+    from utils import datetime_handler
+
+    try:
+        async for result in results_gen:
+            try:
+                result_dict = result.model_dump()
+                logger.info(f"Streaming result for {result_dict.get('url', 'unknown')}")
+                data = json.dumps(result_dict, default=datetime_handler) + "\n"
+                yield data.encode('utf-8')
+            except Exception as e:
+                logger.error(f"Serialization error: {e}")
+                error_response = {"error": str(e), "url": getattr(result, 'url', 'unknown')}
+                yield (json.dumps(error_response) + "\n").encode('utf-8')
+
+        yield json.dumps({"status": "completed"}).encode('utf-8')
+        
+    except asyncio.CancelledError:
+        logger.warning("Client disconnected during streaming")
+    finally:
+        try:
+            await crawler.close()
+        except Exception as e:
+            logger.error(f"Crawler cleanup error: {e}")
+
+async def handle_crawl_request(
+    urls: List[str],
+    browser_config: dict,
+    crawler_config: dict,
+    config: dict
+) -> dict:
+    """Handle non-streaming crawl requests."""
+    try:
+        browser_config = BrowserConfig.load(browser_config)
+        crawler_config = CrawlerRunConfig.load(crawler_config)
+
+        dispatcher = MemoryAdaptiveDispatcher(
+            memory_threshold_percent=config["crawler"]["memory_threshold_percent"],
+            rate_limiter=RateLimiter(
+                base_delay=tuple(config["crawler"]["rate_limiter"]["base_delay"])
+            )
+        )
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            results = await crawler.arun_many(
+                urls=urls,
+                config=crawler_config,
+                dispatcher=dispatcher
+            )
+            
+            return {
+                "success": True,
+                "results": [result.model_dump() for result in results]
+            }
+
+    except Exception as e:
+        logger.error(f"Crawl error: {str(e)}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=str(e)
+        )
+
+async def handle_stream_crawl_request(
+    urls: List[str],
+    browser_config: dict,
+    crawler_config: dict,
+    config: dict
+) -> Tuple[AsyncWebCrawler, AsyncGenerator]:
+    """Handle streaming crawl requests."""
+    try:
+        browser_config = BrowserConfig.load(browser_config)
+        browser_config.verbose = True
+        crawler_config = CrawlerRunConfig.load(crawler_config)
+        crawler_config.scraping_strategy = LXMLWebScrapingStrategy()
+
+        dispatcher = MemoryAdaptiveDispatcher(
+            memory_threshold_percent=config["crawler"]["memory_threshold_percent"],
+            rate_limiter=RateLimiter(
+                base_delay=tuple(config["crawler"]["rate_limiter"]["base_delay"])
+            )
+        )
+
+        crawler = AsyncWebCrawler(config=browser_config)
+        await crawler.start()
+
+        results_gen = await crawler.arun_many(
+            urls=urls,
+            config=crawler_config,
+            dispatcher=dispatcher
+        )
+
+        return crawler, results_gen
+
+    except Exception as e:
+        if 'crawler' in locals():
+            await crawler.close()
+        logger.error(f"Stream crawl error: {str(e)}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=str(e)
+        )

deploy/aws/docker/auth.py

@@ -0,0 +1,46 @@
+import os
+from datetime import datetime, timedelta, timezone
+from typing import Dict, Optional
+from jwt import JWT, jwk_from_dict
+from jwt.utils import get_int_from_datetime
+from fastapi import Depends, HTTPException
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from pydantic import EmailStr
+from pydantic.main import BaseModel
+import base64
+
+instance = JWT()
+security = HTTPBearer()
+SECRET_KEY = os.environ.get("SECRET_KEY", "mysecret")
+ACCESS_TOKEN_EXPIRE_MINUTES = 60
+
+def get_jwk_from_secret(secret: str):
+    """Convert a secret string into a JWK object."""
+    secret_bytes = secret.encode('utf-8')
+    b64_secret = base64.urlsafe_b64encode(secret_bytes).rstrip(b'=').decode('utf-8')
+    return jwk_from_dict({"kty": "oct", "k": b64_secret})
+
+def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) -> str:
+    """Create a JWT access token with an expiration."""
+    to_encode = data.copy()
+    expire = datetime.now(timezone.utc) + (expires_delta or timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES))
+    to_encode.update({"exp": get_int_from_datetime(expire)})
+    signing_key = get_jwk_from_secret(SECRET_KEY)
+    return instance.encode(to_encode, signing_key, alg='HS256')
+
+def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)) -> Dict:
+    """Verify the JWT token from the Authorization header."""
+    token = credentials.credentials
+    verifying_key = get_jwk_from_secret(SECRET_KEY)
+    try:
+        payload = instance.decode(token, verifying_key, do_time_check=True, algorithms='HS256')
+        return payload
+    except Exception:
+        raise HTTPException(status_code=401, detail="Invalid or expired token")
+
+def get_token_dependency(config: Dict):
+    """Return the token dependency if JWT is enabled, else None."""
+    return verify_token if config.get("security", {}).get("jwt_enabled", False) else None
+
+class TokenRequest(BaseModel):
+    email: EmailStr

deploy/aws/docker/config.yml

@@ -0,0 +1,71 @@
+# Application Configuration
+app:
+  title: "Crawl4AI API"
+  version: "1.0.0"
+  host: "0.0.0.0"
+  port: 8000
+  reload: True
+  timeout_keep_alive: 300
+
+# Default LLM Configuration
+llm:
+  provider: "openai/gpt-4o-mini"
+  api_key_env: "OPENAI_API_KEY"
+  # api_key: sk-...  # If you pass the API key directly then api_key_env will be ignored
+
+# Redis Configuration
+redis:
+  host: "localhost"
+  port: 6379
+  db: 0
+  password: ""
+  ssl: False
+  ssl_cert_reqs: None
+  ssl_ca_certs: None
+  ssl_certfile: None
+  ssl_keyfile: None
+  ssl_cert_reqs: None
+  ssl_ca_certs: None
+  ssl_certfile: None
+  ssl_keyfile: None
+
+# Rate Limiting Configuration
+rate_limiting:
+  enabled: True
+  default_limit: "1000/minute"
+  trusted_proxies: []
+  storage_uri: "memory://"  # Use "redis://localhost:6379" for production
+
+# Security Configuration
+security:
+  enabled: true 
+  jwt_enabled: true 
+  https_redirect: false
+  trusted_hosts: ["*"]
+  headers:
+    x_content_type_options: "nosniff"
+    x_frame_options: "DENY"
+    content_security_policy: "default-src 'self'"
+    strict_transport_security: "max-age=63072000; includeSubDomains"
+
+# Crawler Configuration
+crawler:
+  memory_threshold_percent: 95.0
+  rate_limiter:
+    base_delay: [1.0, 2.0]
+  timeouts:
+    stream_init: 30.0  # Timeout for stream initialization
+    batch_process: 300.0  # Timeout for batch processing
+
+# Logging Configuration
+logging:
+  level: "INFO"
+  format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+# Observability Configuration
+observability:
+  prometheus:
+    enabled: True
+    endpoint: "/metrics"
+  health_check:
+    endpoint: "/health"

deploy/aws/docker/requirements.txt

@@ -0,0 +1,10 @@
+crawl4ai
+fastapi
+uvicorn
+gunicorn>=23.0.0
+slowapi>=0.1.9
+prometheus-fastapi-instrumentator>=7.0.2
+redis>=5.2.1
+jwt>=1.3.1
+dnspython>=2.7.0
+email-validator>=2.2.0

deploy/aws/docker/server.py

@@ -0,0 +1,181 @@
+import os
+import sys
+import time
+from typing import List, Optional, Dict
+from fastapi import FastAPI, HTTPException, Request, Query, Path, Depends
+from fastapi.responses import StreamingResponse, RedirectResponse, PlainTextResponse, JSONResponse
+from fastapi.middleware.httpsredirect import HTTPSRedirectMiddleware
+from fastapi.middleware.trustedhost import TrustedHostMiddleware
+from pydantic import BaseModel, Field
+from slowapi import Limiter
+from slowapi.util import get_remote_address
+from prometheus_fastapi_instrumentator import Instrumentator
+from redis import asyncio as aioredis
+
+sys.path.append(os.path.dirname(os.path.realpath(__file__)))
+from utils import FilterType, load_config, setup_logging, verify_email_domain
+from api import (
+    handle_markdown_request,
+    handle_llm_qa,
+    handle_stream_crawl_request,
+    handle_crawl_request,
+    stream_results
+)
+from auth import create_access_token, get_token_dependency, TokenRequest  # Import from auth.py
+
+__version__ = "0.2.6"
+
+class CrawlRequest(BaseModel):
+    urls: List[str] = Field(min_length=1, max_length=100)
+    browser_config: Optional[Dict] = Field(default_factory=dict)
+    crawler_config: Optional[Dict] = Field(default_factory=dict)
+
+# Load configuration and setup
+config = load_config()
+setup_logging(config)
+
+# Initialize Redis
+redis = aioredis.from_url(config["redis"].get("uri", "redis://localhost"))
+
+# Initialize rate limiter
+limiter = Limiter(
+    key_func=get_remote_address,
+    default_limits=[config["rate_limiting"]["default_limit"]],
+    storage_uri=config["rate_limiting"]["storage_uri"]
+)
+
+app = FastAPI(
+    title=config["app"]["title"],
+    version=config["app"]["version"]
+)
+
+# Configure middleware
+def setup_security_middleware(app, config):
+    sec_config = config.get("security", {})
+    if sec_config.get("enabled", False):
+        if sec_config.get("https_redirect", False):
+            app.add_middleware(HTTPSRedirectMiddleware)
+        if sec_config.get("trusted_hosts", []) != ["*"]:
+            app.add_middleware(TrustedHostMiddleware, allowed_hosts=sec_config["trusted_hosts"])
+
+setup_security_middleware(app, config)
+
+# Prometheus instrumentation
+if config["observability"]["prometheus"]["enabled"]:
+    Instrumentator().instrument(app).expose(app)
+
+# Get token dependency based on config
+token_dependency = get_token_dependency(config)
+
+# Middleware for security headers
+@app.middleware("http")
+async def add_security_headers(request: Request, call_next):
+    response = await call_next(request)
+    if config["security"]["enabled"]:
+        response.headers.update(config["security"]["headers"])
+    return response
+
+# Token endpoint (always available, but usage depends on config)
+@app.post("/token")
+async def get_token(request_data: TokenRequest):
+    if not verify_email_domain(request_data.email):
+        raise HTTPException(status_code=400, detail="Invalid email domain")
+    token = create_access_token({"sub": request_data.email})
+    return {"email": request_data.email, "access_token": token, "token_type": "bearer"}
+
+# Endpoints with conditional auth
+@app.get("/md/{url:path}")
+@limiter.limit(config["rate_limiting"]["default_limit"])
+async def get_markdown(
+    request: Request,
+    url: str,
+    f: FilterType = FilterType.FIT,
+    q: Optional[str] = None,
+    c: Optional[str] = "0",
+    token_data: Optional[Dict] = Depends(token_dependency)
+):
+    result = await handle_markdown_request(url, f, q, c, config)
+    return PlainTextResponse(result)
+
+@app.get("/llm/{url:path}", description="URL should be without http/https prefix")
+async def llm_endpoint(
+    request: Request,
+    url: str = Path(...),
+    q: Optional[str] = Query(None),
+    token_data: Optional[Dict] = Depends(token_dependency)
+):
+    if not q:
+        raise HTTPException(status_code=400, detail="Query parameter 'q' is required")
+    if not url.startswith(('http://', 'https://')):
+        url = 'https://' + url
+    try:
+        answer = await handle_llm_qa(url, q, config)
+        return JSONResponse({"answer": answer})
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/schema")
+async def get_schema():
+    from crawl4ai import BrowserConfig, CrawlerRunConfig
+    return {"browser": BrowserConfig().dump(), "crawler": CrawlerRunConfig().dump()}
+
+@app.get(config["observability"]["health_check"]["endpoint"])
+async def health():
+    return {"status": "ok", "timestamp": time.time(), "version": __version__}
+
+@app.get(config["observability"]["prometheus"]["endpoint"])
+async def metrics():
+    return RedirectResponse(url=config["observability"]["prometheus"]["endpoint"])
+
+@app.post("/crawl")
+@limiter.limit(config["rate_limiting"]["default_limit"])
+async def crawl(
+    request: Request,
+    crawl_request: CrawlRequest,
+    token_data: Optional[Dict] = Depends(token_dependency)
+):
+    if not crawl_request.urls:
+        raise HTTPException(status_code=400, detail="At least one URL required")
+    
+    results = await handle_crawl_request(
+        urls=crawl_request.urls,
+        browser_config=crawl_request.browser_config,
+        crawler_config=crawl_request.crawler_config,
+        config=config
+    )
+
+    return JSONResponse(results)
+
+
+@app.post("/crawl/stream")
+@limiter.limit(config["rate_limiting"]["default_limit"])
+async def crawl_stream(
+    request: Request,
+    crawl_request: CrawlRequest,
+    token_data: Optional[Dict] = Depends(token_dependency)
+):
+    if not crawl_request.urls:
+        raise HTTPException(status_code=400, detail="At least one URL required")
+
+    crawler, results_gen = await handle_stream_crawl_request(
+        urls=crawl_request.urls,
+        browser_config=crawl_request.browser_config,
+        crawler_config=crawl_request.crawler_config,
+        config=config
+    )
+
+    return StreamingResponse(
+        stream_results(crawler, results_gen),
+        media_type='application/x-ndjson',
+        headers={'Cache-Control': 'no-cache', 'Connection': 'keep-alive', 'X-Stream-Status': 'active'}
+    )
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(
+        "server:app",
+        host=config["app"]["host"],
+        port=config["app"]["port"],
+        reload=config["app"]["reload"],
+        timeout_keep_alive=config["app"]["timeout_keep_alive"]
+    )

deploy/aws/docker/supervisord.conf

@@ -0,0 +1,12 @@
+[supervisord]
+nodaemon=true
+
+[program:redis]
+command=redis-server
+autorestart=true
+priority=10
+
+[program:gunicorn]
+command=gunicorn --bind 0.0.0.0:8000 --workers 4 --threads 2 --timeout 300 --graceful-timeout 60 --keep-alive 65 --log-level debug --worker-class uvicorn.workers.UvicornWorker --max-requests 1000 --max-requests-jitter 50 server:app
+autorestart=true
+priority=20

deploy/aws/docker/utils.py

@@ -0,0 +1,66 @@
+import dns.resolver
+import logging
+import yaml
+from datetime import datetime
+from enum import Enum
+from pathlib import Path
+from fastapi import Request
+from typing import Dict, Optional
+
+class TaskStatus(str, Enum):
+    PROCESSING = "processing"
+    FAILED = "failed"
+    COMPLETED = "completed"
+
+class FilterType(str, Enum):
+    RAW = "raw"
+    FIT = "fit"
+    BM25 = "bm25"
+    LLM = "llm"
+
+def load_config() -> Dict:
+    """Load and return application configuration."""
+    config_path = Path(__file__).parent / "config.yml"
+    with open(config_path, "r") as config_file:
+        return yaml.safe_load(config_file)
+
+def setup_logging(config: Dict) -> None:
+    """Configure application logging."""
+    logging.basicConfig(
+        level=config["logging"]["level"],
+        format=config["logging"]["format"]
+    )
+
+def get_base_url(request: Request) -> str:
+    """Get base URL including scheme and host."""
+    return f"{request.url.scheme}://{request.url.netloc}"
+
+def is_task_id(value: str) -> bool:
+    """Check if the value matches task ID pattern."""
+    return value.startswith("llm_") and "_" in value
+
+def datetime_handler(obj: any) -> Optional[str]:
+    """Handle datetime serialization for JSON."""
+    if hasattr(obj, 'isoformat'):
+        return obj.isoformat()
+    raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
+
+def should_cleanup_task(created_at: str) -> bool:
+    """Check if task should be cleaned up based on creation time."""
+    created = datetime.fromisoformat(created_at)
+    return (datetime.now() - created).total_seconds() > 3600
+
+def decode_redis_hash(hash_data: Dict[bytes, bytes]) -> Dict[str, str]:
+    """Decode Redis hash data from bytes to strings."""
+    return {k.decode('utf-8'): v.decode('utf-8') for k, v in hash_data.items()}
+
+
+
+def verify_email_domain(email: str) -> bool:
+    try:
+        domain = email.split('@')[1]
+        # Try to resolve MX records for the domain.
+        records = dns.resolver.resolve(domain, 'MX')
+        return True if records else False
+    except Exception as e:
+        return False

deploy/aws/howto.md

@@ -0,0 +1,77 @@
+# Crawl4AI API Quickstart
+
+This document shows how to generate an API token and use it to call the `/crawl` and `/md` endpoints.
+
+---
+
+## 1. Crawl Example
+
+Send a POST request to `/crawl` with the following JSON payload:
+
+```json
+{
+  "urls": ["https://example.com"],
+  "browser_config": { "headless": true, "verbose": true },
+  "crawler_config": { "stream": false, "cache_mode": "enabled" }
+}
+```
+
+**cURL Command:**
+
+```bash
+curl -X POST "https://api.crawl4ai.com/crawl" \
+  -H "Authorization: Bearer YOUR_API_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{
+        "urls": ["https://example.com"],
+        "browser_config": {"headless": true, "verbose": true},
+        "crawler_config": {"stream": false, "cache_mode": "enabled"}
+      }'
+```
+
+---
+
+## 2. Markdown Retrieval Example
+
+To retrieve markdown from a given URL (e.g., `https://example.com`), use:
+
+```bash
+curl -X GET "https://api.crawl4ai.com/md/example.com" \
+  -H "Authorization: Bearer YOUR_API_TOKEN"
+```
+
+---
+
+## 3. Python Code Example (Using `requests`)
+
+Below is a sample Python script that demonstrates using the `requests` library to call the API endpoints:
+
+```python
+import requests
+
+BASE_URL = "https://api.crawl4ai.com"
+TOKEN = "YOUR_API_TOKEN"  # Replace with your actual token
+
+headers = {
+    "Authorization": f"Bearer {TOKEN}",
+    "Content-Type": "application/json"
+}
+
+# Crawl endpoint example
+crawl_payload = {
+    "urls": ["https://example.com"],
+    "browser_config": {"headless": True, "verbose": True},
+    "crawler_config": {"stream": False, "cache_mode": "enabled"}
+}
+
+crawl_response = requests.post(f"{BASE_URL}/crawl", json=crawl_payload, headers=headers)
+print("Crawl Response:", crawl_response.json())
+
+# /md endpoint example
+md_response = requests.get(f"{BASE_URL}/md/example.com", headers=headers)
+print("Markdown Content:", md_response.text)
+```
+
+---
+
+Happy crawling!

deploy/aws/nginx/Dockerfile

@@ -0,0 +1,2 @@
+FROM nginx:alpine
+COPY nginx.conf /etc/nginx/conf.d/default.conf

deploy/aws/nginx/nginx.conf

@@ -0,0 +1,55 @@
+server {
+    listen 80;
+    server_name api.crawl4ai.com;
+
+    # Main logging settings
+    error_log /var/log/nginx/error.log debug;
+    access_log /var/log/nginx/access.log combined buffer=512k flush=1m;
+
+    # Timeout and buffering settings
+    proxy_connect_timeout 300;
+    proxy_send_timeout 300;
+    proxy_read_timeout 300;
+    send_timeout 300;
+    proxy_buffer_size 128k;
+    proxy_buffers 4 256k;
+    proxy_busy_buffers_size 256k;
+
+    # Health check location
+    location /health {
+        proxy_pass http://127.0.0.1:8000/health;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    # Main proxy for application endpoints
+    location / {
+        proxy_pass http://127.0.0.1:8000;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        add_header X-Debug-Info $request_uri;
+        proxy_request_buffering off;
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
+        proxy_buffering off;
+    }
+
+    # New endpoint: serve Nginx error log
+    location /nginx/error {
+        # Using "alias" to serve the error log file
+        alias /var/log/nginx/error.log;
+        # Optionally, you might restrict access with "allow" and "deny" directives.
+    }
+
+    # New endpoint: serve Nginx access log
+    location /nginx/access {
+        alias /var/log/nginx/access.log;
+    }
+
+    client_max_body_size 10M;
+    client_body_buffer_size 128k;
+}

deploy/aws/version.txt

@@ -0,0 +1 @@
+v0.1.0

deploy/docker/README.md

@@ -352,7 +352,10 @@ Example:
 from crawl4ai import CrawlerRunConfig, PruningContentFilter
 
 config = CrawlerRunConfig(
-    content_filter=PruningContentFilter(threshold=0.48)
+    markdown_generator=DefaultMarkdownGenerator(
+        content_filter=PruningContentFilter(threshold=0.48, threshold_type="fixed")
+    ),
+    cache_mode= CacheMode.BYPASS
 )
 print(config.dump())  # Use this JSON in your API calls
 ```

deploy/gcloud-function/Dockerfile

@@ -0,0 +1,63 @@
+FROM --platform=linux/amd64 python:3.10-slim
+
+# Install system dependencies required for Chromium and Git
+RUN apt-get update && apt-get install -y \
+    python3-dev \
+    pkg-config \
+    libjpeg-dev \
+    gcc \
+    build-essential \
+    libnss3 \
+    libnspr4 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    libcups2 \
+    libdrm2 \
+    libxkbcommon0 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxfixes3 \
+    libxrandr2 \
+    libgbm1 \
+    libasound2 \
+    libpango-1.0-0 \
+    libcairo2 \
+    procps \
+    git \
+    socat \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Make a directory for crawl4ai call it crawl4ai_repo
+# RUN mkdir crawl4ai_repo
+
+# # Clone Crawl4ai from the next branch and install it
+# RUN git clone --branch next https://github.com/unclecode/crawl4ai.git ./crawl4ai_repo \
+#     && cd crawl4ai_repo \
+#     && pip install . \
+#     && cd .. \
+#     && rm -rf crawl4ai_repo
+
+RUN python3 -m venv /app/venv
+ENV PATH="/app/venv/bin:$PATH"
+# RUN pip install git+https://github.com/unclecode/crawl4ai.git@next
+   
+# Copy requirements and install remaining dependencies
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+# Copy application files
+COPY resources /app/resources
+COPY main.py .
+COPY start.sh .
+
+# Set permissions for Chrome binary and start script
+RUN chmod +x /app/resources/chrome/headless_shell && \
+    chmod -R 755 /app/resources/chrome && \
+    chmod +x start.sh
+
+ENV FUNCTION_TARGET=crawl
+EXPOSE 8080 9223
+
+CMD ["/app/start.sh"]

deploy/gcloud-function/config.yml

@@ -0,0 +1,8 @@
+project_id: PROJECT_ID
+region: REGION_NAME
+artifact_repo: ARTIFACT_REPO_NAME
+function_name: FUNCTION_NAME
+memory: "2048MB"
+timeout: "540s"
+local_image: "gcr.io/ARTIFACT_REPO_NAME/crawl4ai:latest"
+test_query_url: "https://example.com"

deploy/gcloud-function/deploy.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+import argparse
+import subprocess
+import sys
+import yaml
+import requests
+
+def run_command(cmd, explanation, require_confirm=True, allow_already_exists=False):
+    print("\n=== {} ===".format(explanation))
+    if require_confirm:
+        input("Press Enter to run: [{}]\n".format(cmd))
+    print("Running: {}".format(cmd))
+    result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
+    if result.returncode != 0:
+        if allow_already_exists and "ALREADY_EXISTS" in result.stderr:
+            print("Repository already exists, skipping creation.")
+            return ""
+        print("Error:\n{}".format(result.stderr))
+        sys.exit(1)
+    out = result.stdout.strip()
+    if out:
+        print("Output:\n{}".format(out))
+    return out
+
+def load_config():
+    try:
+        with open("config.yml", "r") as f:
+            config = yaml.safe_load(f)
+    except Exception as e:
+        print("Failed to load config.yml: {}".format(e))
+        sys.exit(1)
+    required = ["project_id", "region", "artifact_repo", "function_name", "local_image"]
+    for key in required:
+        if key not in config or not config[key]:
+            print("Missing required config parameter: {}".format(key))
+            sys.exit(1)
+    return config
+
+def deploy_function(config):
+    project_id     = config["project_id"]
+    region         = config["region"]
+    artifact_repo  = config["artifact_repo"]
+    function_name  = config["function_name"]
+    memory         = config.get("memory", "2048MB")
+    timeout        = config.get("timeout", "540s")
+    local_image    = config["local_image"]
+    test_query_url = config.get("test_query_url", "https://example.com")
+
+    # Repository image format: "<region>-docker.pkg.dev/<project_id>/<artifact_repo>/<function_name>:latest"
+    repo_image = f"{region}-docker.pkg.dev/{project_id}/{artifact_repo}/{function_name}:latest"
+
+    # 1. Create Artifact Registry repository (skip if exists)
+    cmd = f"gcloud artifacts repositories create {artifact_repo} --repository-format=docker --location={region} --project={project_id}"
+    run_command(cmd, "Creating Artifact Registry repository (if it doesn't exist)", allow_already_exists=True)
+
+    # 2. Tag the local Docker image with the repository image name
+    cmd = f"docker tag {local_image} {repo_image}"
+    run_command(cmd, "Tagging Docker image for Artifact Registry")
+
+    # 3. Authenticate Docker to Artifact Registry
+    cmd = f"gcloud auth configure-docker {region}-docker.pkg.dev"
+    run_command(cmd, "Authenticating Docker to Artifact Registry")
+
+    # 4. Push the tagged Docker image to Artifact Registry
+    cmd = f"docker push {repo_image}"
+    run_command(cmd, "Pushing Docker image to Artifact Registry")
+
+    # 5. Deploy the Cloud Function using the custom container
+    cmd = (
+        f"gcloud beta functions deploy {function_name} "
+        f"--gen2 "
+        f"--runtime=python310 "
+        f"--entry-point=crawl "
+        f"--region={region} "
+        f"--docker-repository={region}-docker.pkg.dev/{project_id}/{artifact_repo} "
+        f"--trigger-http "
+        f"--memory={memory} "
+        f"--timeout={timeout} "
+        f"--project={project_id}"
+    )
+    run_command(cmd, "Deploying Cloud Function using custom container")
+
+    # 6. Set the Cloud Function to allow public (unauthenticated) invocations
+    cmd = (
+        f"gcloud functions add-iam-policy-binding {function_name} "
+        f"--region={region} "
+        f"--member='allUsers' "
+        f"--role='roles/cloudfunctions.invoker' "
+        f"--project={project_id}"
+        f"--quiet"
+    )
+    run_command(cmd, "Setting Cloud Function IAM to allow public invocations")
+
+    # 7. Retrieve the deployed Cloud Function URL
+    cmd = (
+        f"gcloud functions describe {function_name} "
+        f"--region={region} "
+        f"--project={project_id} "
+        f"--format='value(serviceConfig.uri)'"
+    )
+    deployed_url = run_command(cmd, "Extracting deployed Cloud Function URL", require_confirm=False)
+    print("\nDeployed URL: {}\n".format(deployed_url))
+
+    # 8. Test the deployed function
+    test_url = f"{deployed_url}?url={test_query_url}"
+    print("Testing function with: {}".format(test_url))
+    try:
+        response = requests.get(test_url)
+        print("Response status: {}".format(response.status_code))
+        print("Response body:\n{}".format(response.text))
+        if response.status_code == 200:
+            print("Test successful!")
+        else:
+            print("Non-200 response; check function logs.")
+    except Exception as e:
+        print("Test request error: {}".format(e))
+        sys.exit(1)
+
+    # 9. Final usage help
+    print("\nDeployment complete!")
+    print("Invoke your function with:")
+    print(f"curl '{deployed_url}?url={test_query_url}'")
+    print("For further instructions, refer to your documentation.")
+
+def delete_function(config):
+    project_id    = config["project_id"]
+    region        = config["region"]
+    function_name = config["function_name"]
+    cmd = f"gcloud functions delete {function_name} --region={region} --project={project_id} --quiet"
+    run_command(cmd, "Deleting Cloud Function")
+
+def describe_function(config):
+    project_id    = config["project_id"]
+    region        = config["region"]
+    function_name = config["function_name"]
+    cmd = (
+        f"gcloud functions describe {function_name} "
+        f"--region={region} "
+        f"--project={project_id} "
+        f"--format='value(serviceConfig.uri)'"
+    )
+    deployed_url = run_command(cmd, "Describing Cloud Function to extract URL", require_confirm=False)
+    print("\nCloud Function URL: {}\n".format(deployed_url))
+
+def clear_all(config):
+    print("\n=== CLEAR ALL RESOURCES ===")
+    project_id    = config["project_id"]
+    region        = config["region"]
+    artifact_repo = config["artifact_repo"]
+
+    confirm = input("WARNING: This will DELETE the Cloud Function and the Artifact Registry repository. Are you sure? (y/N): ")
+    if confirm.lower() != "y":
+        print("Aborting clear operation.")
+        sys.exit(0)
+
+    # Delete the Cloud Function
+    delete_function(config)
+    # Delete the Artifact Registry repository
+    cmd = f"gcloud artifacts repositories delete {artifact_repo} --location={region} --project={project_id} --quiet"
+    run_command(cmd, "Deleting Artifact Registry repository", require_confirm=False)
+    print("All resources cleared.")
+
+def main():
+    parser = argparse.ArgumentParser(description="Deploy, delete, describe, or clear Cloud Function resources using config.yml")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    subparsers.add_parser("deploy", help="Deploy the Cloud Function")
+    subparsers.add_parser("delete", help="Delete the deployed Cloud Function")
+    subparsers.add_parser("describe", help="Describe the Cloud Function and return its URL")
+    subparsers.add_parser("clear", help="Delete the Cloud Function and Artifact Registry repository")
+
+    args = parser.parse_args()
+    config = load_config()
+
+    if args.command == "deploy":
+        deploy_function(config)
+    elif args.command == "delete":
+        delete_function(config)
+    elif args.command == "describe":
+        describe_function(config)
+    elif args.command == "clear":
+        clear_all(config)
+    else:
+        parser.print_help()
+
+if __name__ == "__main__":
+    main()

deploy/gcloud-function/guide.md

@@ -0,0 +1,204 @@
+# Deploying Crawl4ai on Google Cloud Functions
+
+This guide explains how to deploy **Crawl4ai**—an open‑source web crawler library—on Google Cloud Functions Gen2 using a custom container. We assume your project folder already includes:
+
+- **Dockerfile:** Builds your container image (which installs Crawl4ai from its Git repository).
+- **start.sh:** Activates your virtual environment and starts the function (using the Functions Framework).
+- **main.py:** Contains your function logic with the entry point `crawl` (and imports Crawl4ai).
+
+The guide is divided into two parts:
+1. Manual deployment steps (using CLI commands)
+2. Automated deployment using a Python script (`deploy.py`)
+
+---
+
+## Part 1: Manual Deployment Process
+
+### Prerequisites
+
+- **Google Cloud Project:** Ensure your project is active and billing is enabled.
+- **Google Cloud CLI & Docker:** Installed and configured on your local machine.
+- **Permissions:** You must have rights to create Cloud Functions and Artifact Registry repositories.
+- **Files:** Your Dockerfile, start.sh, and main.py should be in the same directory.
+
+### Step 1: Build Your Docker Image
+
+Your Dockerfile packages Crawl4ai along with all its dependencies. Build your image with:
+
+```bash
+docker build -t gcr.io/<PROJECT_ID>/<FUNCTION_NAME>:latest .
+```
+
+Replace `<PROJECT_ID>` with your Google Cloud project ID and `<FUNCTION_NAME>` with your chosen function name (for example, `crawl4ai-t1`).
+
+### Step 2: Create an Artifact Registry Repository
+
+Cloud Functions Gen2 requires your custom container image to reside in an Artifact Registry repository. Create one by running:
+
+```bash
+gcloud artifacts repositories create <ARTIFACT_REPO> \
+  --repository-format=docker \
+  --location=<REGION> \
+  --project=<PROJECT_ID>
+```
+
+Replace `<ARTIFACT_REPO>` (for example, `crawl4ai`) and `<REGION>` (for example, `asia-east1`).  
+> **Note:** If you receive an `ALREADY_EXISTS` error, the repository is already created; simply proceed to the next step.
+
+### Step 3: Tag Your Docker Image
+
+Tag your locally built Docker image so it matches the Artifact Registry format:
+
+```bash
+docker tag gcr.io/<PROJECT_ID>/<FUNCTION_NAME>:latest <REGION>-docker.pkg.dev/<PROJECT_ID>/<ARTIFACT_REPO>/<FUNCTION_NAME>:latest
+```
+
+This step “renames” the image so you can push it to your repository.
+
+### Step 4: Authenticate Docker to Artifact Registry
+
+Configure Docker authentication to the Artifact Registry:
+
+```bash
+gcloud auth configure-docker <REGION>-docker.pkg.dev
+```
+
+This ensures Docker can securely push images to your registry using your Cloud credentials.
+
+### Step 5: Push the Docker Image
+
+Push the tagged image to Artifact Registry:
+
+```bash
+docker push <REGION>-docker.pkg.dev/<PROJECT_ID>/<ARTIFACT_REPO>/<FUNCTION_NAME>:latest
+```
+
+Once complete, your container image (with Crawl4ai installed) is hosted in Artifact Registry.
+
+### Step 6: Deploy the Cloud Function
+
+Deploy your function using the custom container image. Run:
+
+```bash
+gcloud beta functions deploy <FUNCTION_NAME> \
+  --gen2 \
+  --region=<REGION> \
+  --docker-repository=<REGION>-docker.pkg.dev/<PROJECT_ID>/<ARTIFACT_REPO> \
+  --trigger-http \
+  --memory=2048MB \
+  --timeout=540s \
+  --project=<PROJECT_ID>
+```
+
+This command tells Cloud Functions Gen2 to pull your container image from Artifact Registry and deploy it. Make sure your main.py defines the `crawl` entry point.
+
+### Step 7: Make the Function Public
+
+To allow external (unauthenticated) access, update the function’s IAM policy:
+
+```bash
+gcloud functions add-iam-policy-binding <FUNCTION_NAME> \
+  --region=<REGION> \
+  --member="allUsers" \
+  --role="roles/cloudfunctions.invoker" \
+  --project=<PROJECT_ID> \
+  --quiet
+```
+
+Using the `--quiet` flag ensures the command runs non‑interactively so the policy is applied immediately.
+
+### Step 8: Retrieve and Test Your Function URL
+
+Get the URL for your deployed function:
+
+```bash
+gcloud functions describe <FUNCTION_NAME> \
+  --region=<REGION> \
+  --project=<PROJECT_ID> \
+  --format='value(serviceConfig.uri)'
+```
+
+Test your deployment with a sample GET request (using curl or your browser):
+
+```bash
+curl "<FUNCTION_URL>?url=https://example.com"
+```
+
+Replace `<FUNCTION_URL>` with the output URL from the previous command. A successful test (HTTP status 200) means Crawl4ai is running on Cloud Functions.
+
+---
+
+## Part 2: Automated Deployment with deploy.py
+
+For a more streamlined process, use the provided `deploy.py` script. This Python script automates the manual steps, prompting you to confirm key actions and providing detailed logs throughout the process.
+
+### What deploy.py Does:
+
+- **Reads Parameters:** It loads a `config.yml` file containing all necessary parameters such as `project_id`, `region`, `artifact_repo`, `function_name`, `local_image`, etc.
+- **Creates/Skips Repository:** It creates the Artifact Registry repository (or skips if it already exists).
+- **Tags & Pushes:** It tags your local Docker image and pushes it to the Artifact Registry.
+- **Deploys the Function:** It deploys the Cloud Function with your custom container.
+- **Updates IAM:** It sets the IAM policy to allow public access (using the `--quiet` flag).
+- **Tests the Deployment:** It extracts the deployed URL and performs a test request.
+- **Additional Commands:** You can also use subcommands in the script to delete or describe the deployed function, or even clear all resources.
+
+### Example config.yml
+
+Create a `config.yml` file in the same folder as your Dockerfile. An example configuration:
+
+```yaml
+project_id: your-project-id
+region: asia-east1
+artifact_repo: crawl4ai
+function_name: crawl4ai-t1
+memory: "2048MB"
+timeout: "540s"
+local_image: "gcr.io/your-project-id/crawl4ai-t1:latest"
+test_query_url: "https://example.com"
+```
+
+### How to Use deploy.py
+
+- **Deploy the Function:**
+
+  ```bash
+  python deploy.py deploy
+  ```
+
+  The script will guide you through each step, display the output, and ask for confirmation before executing critical commands.
+
+- **Describe the Function:**
+
+  If you forget the function URL and want to retrieve it later:
+
+  ```bash
+  python deploy.py describe
+  ```
+
+- **Delete the Function:**
+
+  To remove just the Cloud Function:
+
+  ```bash
+  python deploy.py delete
+  ```
+
+- **Clear All Resources:**
+
+  To delete both the Cloud Function and the Artifact Registry repository:
+
+  ```bash
+  python deploy.py clear
+  ```
+
+---
+
+## Conclusion
+
+This guide has walked you through two deployment methods for Crawl4ai on Google Cloud Functions Gen2:
+
+1. **Manual Deployment:** Building your Docker image, pushing it to Artifact Registry, deploying the Cloud Function, and setting up IAM.
+2. **Automated Deployment:** Using `deploy.py` with a configuration file to handle the entire process interactively.
+
+By following these instructions, you can deploy, test, and manage your Crawl4ai-based Cloud Function with ease. Enjoy using Crawl4ai in your cloud environment!
+

deploy/gcloud-function/main.py

@@ -0,0 +1,158 @@
+# Cleanup Chrome process on module unload
+import atexit
+import asyncio
+import logging
+import functions_framework
+from flask import jsonify, Request
+import os
+import sys
+import time
+import subprocess
+import signal
+import requests
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+logger.info(f"Python version: {sys.version}")
+logger.info(f"Python path: {sys.path}")
+
+# Try to find where crawl4ai is coming from
+try:
+    import crawl4ai
+    logger.info(f"Crawl4AI module location: {crawl4ai.__file__}")
+    logger.info(f"Contents of crawl4ai: {dir(crawl4ai)}")
+except ImportError:
+    logger.error("Crawl4AI module not found")
+
+# Now attempt the import
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, CrawlResult
+
+# Configure logging
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+# Paths and constants
+FUNCTION_DIR = os.path.dirname(os.path.realpath(__file__))
+CHROME_BINARY = os.path.join(FUNCTION_DIR, "resources/chrome/headless_shell")
+CDP_PORT = 9222
+
+def start_chrome():
+    """Start Chrome process synchronously with exponential backoff."""
+    logger.debug("Starting Chrome process...")
+    chrome_args = [
+        CHROME_BINARY,
+        f"--remote-debugging-port={CDP_PORT}",
+        "--remote-debugging-address=0.0.0.0",
+        "--no-sandbox",
+        "--disable-setuid-sandbox",
+        "--headless=new",
+        "--disable-gpu",
+        "--disable-dev-shm-usage",
+        "--no-zygote",
+        "--single-process",
+        "--disable-features=site-per-process",
+        "--no-first-run",
+        "--disable-extensions"
+    ]
+    
+    process = subprocess.Popen(
+        chrome_args,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        preexec_fn=os.setsid
+    )
+    
+    logger.debug(f"Chrome process started with PID: {process.pid}")
+    
+    # Wait for CDP endpoint with exponential backoff
+    wait_time = 1  # Start with 1 second
+    max_wait_time = 16  # Cap at 16 seconds per retry
+    max_attempts = 10  # Total attempts
+    for attempt in range(max_attempts):
+        try:
+            response = requests.get(f"http://127.0.0.1:{CDP_PORT}/json/version", timeout=2)
+            if response.status_code == 200:
+                # Get ws URL from response
+                ws_url = response.json()['webSocketDebuggerUrl']
+                logger.debug("Chrome CDP is ready")
+                logger.debug(f"CDP URL: {ws_url}")
+                return process
+        except requests.exceptions.ConnectionError:
+            logger.debug(f"Waiting for CDP endpoint (attempt {attempt + 1}/{max_attempts}), retrying in {wait_time} seconds")
+            time.sleep(wait_time)
+            wait_time = min(wait_time * 2, max_wait_time)  # Double wait time, up to max
+    
+    # If we get here, all retries failed
+    stdout, stderr = process.communicate()  # Get output for debugging
+    logger.error(f"Chrome stdout: {stdout.decode()}")
+    logger.error(f"Chrome stderr: {stderr.decode()}")
+    raise Exception("Chrome CDP endpoint failed to start after retries")
+
+async def fetch_with_crawl4ai(url: str) -> dict:
+    """Fetch page content using Crawl4ai and return the result object"""
+    # Get CDP URL from the running Chrome instance
+    version_response = requests.get(f'http://localhost:{CDP_PORT}/json/version')
+    cdp_url = version_response.json()['webSocketDebuggerUrl']
+    
+    # Configure and run Crawl4ai
+    browser_config = BrowserConfig(cdp_url=cdp_url, use_managed_browser=True)
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        crawler_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+        )
+        result : CrawlResult = await crawler.arun(
+            url=url, config=crawler_config
+        )
+        return result.model_dump()  # Convert Pydantic model to dict for JSON response
+
+# Start Chrome when the module loads
+logger.info("Starting Chrome process on module load")
+chrome_process = start_chrome()
+
+@functions_framework.http
+def crawl(request: Request):
+    """HTTP Cloud Function to fetch web content using Crawl4ai"""
+    try:
+        url = request.args.get('url')
+        if not url:
+            return jsonify({'error': 'URL parameter is required', 'status': 400}), 400
+        
+        # Create and run an asyncio event loop
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        
+        try:
+            result = loop.run_until_complete(
+                asyncio.wait_for(fetch_with_crawl4ai(url), timeout=10.0)
+            )
+            return jsonify({
+                'status': 200,
+                'data': result
+            })
+        finally:
+            loop.close()
+            
+    except Exception as e:
+        error_msg = f"Unexpected error: {str(e)}"
+        logger.error(error_msg, exc_info=True)
+        return jsonify({
+            'error': error_msg,
+            'status': 500,
+            'details': {
+                'error_type': type(e).__name__,
+                'stack_trace': str(e),
+                'chrome_running': chrome_process.poll() is None if chrome_process else False
+            }
+        }), 500
+
+
+@atexit.register
+def cleanup():
+    """Cleanup Chrome process on shutdown"""
+    if chrome_process and chrome_process.poll() is None:
+        try:
+            os.killpg(os.getpgid(chrome_process.pid), signal.SIGTERM)
+            logger.info("Chrome process terminated")
+        except Exception as e:
+            logger.error(f"Failed to terminate Chrome process: {e}")

deploy/gcloud-function/requirements.txt

@@ -0,0 +1,5 @@
+functions-framework==3.*
+flask==2.3.3
+requests==2.31.0
+websockets==12.0
+git+https://github.com/unclecode/crawl4ai.git@next

deploy/gcloud-function/resources/chrome/fonts.conf

@@ -0,0 +1,10 @@
+<?xml version="1.0" ?>
+<!DOCTYPE fontconfig SYSTEM "fonts.dtd">
+<fontconfig>
+  <dir>/var/task/.fonts</dir>
+  <dir>/var/task/fonts</dir>
+  <dir>/opt/fonts</dir>
+  <dir>/tmp/fonts</dir>
+  <cachedir>/tmp/fonts-cache/</cachedir>
+  <config></config>
+</fontconfig>

deploy/gcloud-function/resources/chrome/vk_swiftshader_icd.json

@@ -0,0 +1 @@
+{"file_format_version": "1.0.0", "ICD": {"library_path": "./libvk_swiftshader.so", "api_version": "1.0.5"}}

deploy/lambda/Dockerfile

@@ -0,0 +1,104 @@
+FROM python:3.12-bookworm AS python-builder
+
+RUN pip install poetry
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
+
+WORKDIR /app
+
+COPY pyproject.toml poetry.lock ./
+RUN --mount=type=cache,target=$POETRY_CACHE_DIR poetry export -f requirements.txt -o requirements.txt
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    python3-dev \
+    python3-setuptools \
+    python3-wheel \
+    python3-pip \
+    gcc \
+    g++ \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install specific dependencies that have build issues
+RUN pip install --no-cache-dir cchardet
+
+FROM python:3.12-bookworm
+
+# Install AWS Lambda Runtime Interface Client
+RUN python3 -m pip install --no-cache-dir awslambdaric
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    curl \
+    wget \
+    gnupg \
+    git \
+    cmake \
+    pkg-config \
+    python3-dev \
+    libjpeg-dev \
+    redis-server \
+    supervisor \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libglib2.0-0 \
+    libnss3 \
+    libnspr4 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    libcups2 \
+    libdrm2 \
+    libdbus-1-3 \
+    libxcb1 \
+    libxkbcommon0 \
+    libx11-6 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxext6 \
+    libxfixes3 \
+    libxrandr2 \
+    libgbm1 \
+    libpango-1.0-0 \
+    libcairo2 \
+    libasound2 \
+    libatspi2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install build essentials for any compilations needed
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    python3-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set up function directory and browser path
+ARG FUNCTION_DIR="/function"
+RUN mkdir -p "${FUNCTION_DIR}/pw-browsers"
+RUN mkdir -p "/tmp/.crawl4ai"
+
+# Set critical environment variables
+ENV PLAYWRIGHT_BROWSERS_PATH="${FUNCTION_DIR}/pw-browsers" \
+    HOME="/tmp" \
+    CRAWL4_AI_BASE_DIRECTORY="/tmp/.crawl4ai"
+
+# Create Craw4ai base directory
+RUN mkdir -p ${CRAWL4_AI_BASE_DIRECTORY}
+
+RUN pip install --no-cache-dir faust-cchardet
+
+# Install Crawl4ai and dependencies
+RUN pip install --no-cache-dir git+https://github.com/unclecode/crawl4ai.git@next
+
+# Install Chromium only (no deps flag)
+RUN playwright install chromium
+
+# Copy function code
+COPY lambda_function.py ${FUNCTION_DIR}/
+
+# Set working directory
+WORKDIR ${FUNCTION_DIR}
+
+ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ]
+CMD [ "lambda_function.handler" ]

deploy/lambda/guide.md

@@ -0,0 +1,345 @@
+# Deploying Crawl4ai on AWS Lambda
+
+This guide walks you through deploying Crawl4ai as an AWS Lambda function with API Gateway integration. You'll learn how to set up, test, and clean up your deployment.
+
+## Prerequisites
+
+Before you begin, ensure you have:
+
+- AWS CLI installed and configured (`aws configure`)
+- Docker installed and running
+- Python 3.8+ installed
+- Basic familiarity with AWS services
+
+## Project Files
+
+Your project directory should contain:
+
+- `Dockerfile`: Container configuration for Lambda
+- `lambda_function.py`: Lambda handler code
+- `deploy.py`: Our deployment script
+
+## Step 1: Install Required Python Packages
+
+Install the Python packages needed for our deployment script:
+
+```bash
+pip install typer rich
+```
+
+## Step 2: Run the Deployment Script
+
+Our Python script automates the entire deployment process:
+
+```bash
+python deploy.py
+```
+
+The script will guide you through:
+
+1. Configuration setup (AWS region, function name, memory allocation)
+2. Docker image building
+3. ECR repository creation
+4. Lambda function deployment
+5. API Gateway setup
+6. Provisioned concurrency configuration (optional)
+
+Follow the prompts and confirm each step by pressing Enter.
+
+## Step 3: Manual Deployment (Alternative to the Script)
+
+If you prefer to deploy manually or understand what the script does, follow these steps:
+
+### Building and Pushing the Docker Image
+
+```bash
+# Build the Docker image
+docker build -t crawl4ai-lambda .
+
+# Create an ECR repository (if it doesn't exist)
+aws ecr create-repository --repository-name crawl4ai-lambda
+
+# Get ECR login password and login
+aws ecr get-login-password | docker login --username AWS --password-stdin $(aws sts get-caller-identity --query Account --output text).dkr.ecr.us-east-1.amazonaws.com
+
+# Tag the image
+ECR_URI=$(aws ecr describe-repositories --repository-names crawl4ai-lambda --query 'repositories[0].repositoryUri' --output text)
+docker tag crawl4ai-lambda:latest $ECR_URI:latest
+
+# Push the image to ECR
+docker push $ECR_URI:latest
+```
+
+### Creating the Lambda Function
+
+```bash
+# Get IAM role ARN (create it if needed)
+ROLE_ARN=$(aws iam get-role --role-name lambda-execution-role --query 'Role.Arn' --output text)
+
+# Create Lambda function
+aws lambda create-function \
+    --function-name crawl4ai-function \
+    --package-type Image \
+    --code ImageUri=$ECR_URI:latest \
+    --role $ROLE_ARN \
+    --timeout 300 \
+    --memory-size 4096 \
+    --ephemeral-storage Size=10240 \
+    --environment "Variables={CRAWL4_AI_BASE_DIRECTORY=/tmp/.crawl4ai,HOME=/tmp,PLAYWRIGHT_BROWSERS_PATH=/function/pw-browsers}"
+```
+
+If you're updating an existing function:
+
+```bash
+# Update function code
+aws lambda update-function-code \
+    --function-name crawl4ai-function \
+    --image-uri $ECR_URI:latest
+
+# Update function configuration
+aws lambda update-function-configuration \
+    --function-name crawl4ai-function \
+    --timeout 300 \
+    --memory-size 4096 \
+    --ephemeral-storage Size=10240 \
+    --environment "Variables={CRAWL4_AI_BASE_DIRECTORY=/tmp/.crawl4ai,HOME=/tmp,PLAYWRIGHT_BROWSERS_PATH=/function/pw-browsers}"
+```
+
+### Setting Up API Gateway
+
+```bash
+# Create API Gateway
+API_ID=$(aws apigateway create-rest-api --name crawl4ai-api --query 'id' --output text)
+
+# Get root resource ID
+PARENT_ID=$(aws apigateway get-resources --rest-api-id $API_ID --query 'items[?path==`/`].id' --output text)
+
+# Create resource
+RESOURCE_ID=$(aws apigateway create-resource --rest-api-id $API_ID --parent-id $PARENT_ID --path-part "crawl" --query 'id' --output text)
+
+# Create POST method
+aws apigateway put-method --rest-api-id $API_ID --resource-id $RESOURCE_ID --http-method POST --authorization-type NONE
+
+# Get Lambda function ARN
+LAMBDA_ARN=$(aws lambda get-function --function-name crawl4ai-function --query 'Configuration.FunctionArn' --output text)
+
+# Set Lambda integration
+aws apigateway put-integration \
+    --rest-api-id $API_ID \
+    --resource-id $RESOURCE_ID \
+    --http-method POST \
+    --type AWS_PROXY \
+    --integration-http-method POST \
+    --uri arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/$LAMBDA_ARN/invocations
+
+# Deploy API
+aws apigateway create-deployment --rest-api-id $API_ID --stage-name prod
+
+# Set Lambda permission
+ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+aws lambda add-permission \
+    --function-name crawl4ai-function \
+    --statement-id apigateway \
+    --action lambda:InvokeFunction \
+    --principal apigateway.amazonaws.com \
+    --source-arn "arn:aws:execute-api:us-east-1:$ACCOUNT_ID:$API_ID/*/POST/crawl"
+```
+
+### Setting Up Provisioned Concurrency (Optional)
+
+This reduces cold starts:
+
+```bash
+# Publish a version
+VERSION=$(aws lambda publish-version --function-name crawl4ai-function --query 'Version' --output text)
+
+# Create alias
+aws lambda create-alias \
+    --function-name crawl4ai-function \
+    --name prod \
+    --function-version $VERSION
+
+# Configure provisioned concurrency
+aws lambda put-provisioned-concurrency-config \
+    --function-name crawl4ai-function \
+    --qualifier prod \
+    --provisioned-concurrent-executions 2
+
+# Update API Gateway to use alias
+LAMBDA_ALIAS_ARN="arn:aws:lambda:us-east-1:$ACCOUNT_ID:function:crawl4ai-function:prod"
+aws apigateway put-integration \
+    --rest-api-id $API_ID \
+    --resource-id $RESOURCE_ID \
+    --http-method POST \
+    --type AWS_PROXY \
+    --integration-http-method POST \
+    --uri arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/$LAMBDA_ALIAS_ARN/invocations
+
+# Redeploy API Gateway
+aws apigateway create-deployment --rest-api-id $API_ID --stage-name prod
+```
+
+## Step 4: Testing the Deployment
+
+Once deployed, test your function with:
+
+```bash
+ENDPOINT_URL="https://$API_ID.execute-api.us-east-1.amazonaws.com/prod/crawl"
+
+# Test with curl
+curl -X POST $ENDPOINT_URL \
+  -H "Content-Type: application/json" \
+  -d '{"url":"https://example.com"}'
+```
+
+Or using Python:
+
+```python
+import requests
+import json
+
+url = "https://your-api-id.execute-api.us-east-1.amazonaws.com/prod/crawl"
+payload = {
+    "url": "https://example.com",
+    "browser_config": {
+        "headless": True,
+        "verbose": False
+    },
+    "crawler_config": {
+        "crawler_config": {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "markdown_generator": {
+                    "type": "DefaultMarkdownGenerator",
+                    "params": {
+                        "content_filter": {
+                            "type": "PruningContentFilter",
+                            "params": {
+                                "threshold": 0.48,
+                                "threshold_type": "fixed"
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+response = requests.post(url, json=payload)
+result = response.json()
+print(json.dumps(result, indent=2))
+```
+
+## Step 5: Cleaning Up Resources
+
+To remove all AWS resources created for this deployment:
+
+```bash
+python deploy.py cleanup
+```
+
+Or manually:
+
+```bash
+# Delete API Gateway
+aws apigateway delete-rest-api --rest-api-id $API_ID
+
+# Remove provisioned concurrency (if configured)
+aws lambda delete-provisioned-concurrency-config \
+    --function-name crawl4ai-function \
+    --qualifier prod
+
+# Delete alias (if created)
+aws lambda delete-alias \
+    --function-name crawl4ai-function \
+    --name prod
+
+# Delete Lambda function
+aws lambda delete-function --function-name crawl4ai-function
+
+# Delete ECR repository
+aws ecr delete-repository --repository-name crawl4ai-lambda --force
+```
+
+## Troubleshooting
+
+### Cold Start Issues
+
+If experiencing long cold starts:
+- Enable provisioned concurrency
+- Increase memory allocation (4096 MB recommended)
+- Ensure the Lambda function has enough ephemeral storage
+
+### Permission Errors
+
+If you encounter permission errors:
+- Check the IAM role has the necessary permissions
+- Ensure API Gateway has permission to invoke the Lambda function
+
+### Container Size Issues
+
+If your container is too large:
+- Optimize the Dockerfile
+- Use multi-stage builds
+- Consider removing unnecessary dependencies
+
+## Performance Considerations
+
+- Lambda memory affects CPU allocation - higher memory means faster execution
+- Provisioned concurrency eliminates cold starts but costs more
+- Optimize the Playwright setup for faster browser initialization
+
+## Security Best Practices
+
+- Use the principle of least privilege for IAM roles
+- Implement API Gateway authentication for production deployments
+- Consider using AWS KMS for storing sensitive configuration
+
+## Useful AWS Console Links
+
+Here are quick links to access important AWS console pages for monitoring and managing your deployment:
+
+| Resource | Console Link |
+|----------|-------------|
+| Lambda Functions | [AWS Lambda Console](https://console.aws.amazon.com/lambda/home#/functions) |
+| Lambda Function Logs | [CloudWatch Logs](https://console.aws.amazon.com/cloudwatch/home#logsV2:log-groups) |
+| API Gateway | [API Gateway Console](https://console.aws.amazon.com/apigateway/home) |
+| ECR Repositories | [ECR Console](https://console.aws.amazon.com/ecr/repositories) |
+| IAM Roles | [IAM Console](https://console.aws.amazon.com/iamv2/home#/roles) |
+| CloudWatch Metrics | [CloudWatch Metrics](https://console.aws.amazon.com/cloudwatch/home#metricsV2) |
+
+### Monitoring Lambda Execution
+
+To monitor your Lambda function:
+
+1. Go to the [Lambda function console](https://console.aws.amazon.com/lambda/home#/functions)
+2. Select your function (`crawl4ai-function`)
+3. Click the "Monitor" tab to see:
+   - Invocation metrics
+   - Success/failure rates
+   - Duration statistics
+
+### Viewing Lambda Logs
+
+To see detailed execution logs:
+
+1. Go to [CloudWatch Logs](https://console.aws.amazon.com/cloudwatch/home#logsV2:log-groups)
+2. Find the log group named `/aws/lambda/crawl4ai-function`
+3. Click to see the latest log streams
+4. Each stream contains logs from a function execution
+
+### Checking API Gateway Traffic
+
+To monitor API requests:
+
+1. Go to the [API Gateway console](https://console.aws.amazon.com/apigateway/home)
+2. Select your API (`crawl4ai-api`)
+3. Click "Dashboard" to see:
+   - API calls
+   - Latency
+   - Error rates
+
+## Conclusion
+
+You now have Crawl4ai running as a serverless function on AWS Lambda! This setup allows you to crawl websites on-demand without maintaining infrastructure, while paying only for the compute time you use.

deploy/lambda/lambda_function.py

@@ -0,0 +1,107 @@
+import json
+import asyncio
+import os
+
+# Ensure environment variables and directories are set
+os.environ['CRAWL4_AI_BASE_DIRECTORY'] = '/tmp/.crawl4ai'
+os.environ['HOME'] = '/tmp'
+
+# Create directory if it doesn't exist
+os.makedirs('/tmp/.crawl4ai', exist_ok=True)
+
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig,
+    CrawlerRunConfig,
+    CacheMode
+)
+
+
+def handler(event, context):
+    # Parse the incoming event (API Gateway request)
+    try:
+        body = json.loads(event.get('body', '{}'))
+        
+        url = body.get('url')
+        if not url:
+            return {
+                'statusCode': 400,
+                'body': json.dumps({'error': 'URL is required'})
+            }
+        
+        # Get optional configurations or use defaults
+        browser_config_dict = body.get('browser_config', {})
+        crawler_config_dict = body.get('crawler_config', {})
+        
+        # Run the crawler
+        result = asyncio.run(crawl(url, browser_config_dict, crawler_config_dict))
+        
+        # Return successful response
+        return {
+            'statusCode': 200,
+            'headers': {
+                'Content-Type': 'application/json'
+            },
+            'body': json.dumps(result)
+        }
+    
+    except Exception as e:
+        # Handle errors
+        import traceback
+        return {
+            'statusCode': 500,
+            'body': json.dumps({
+                'error': str(e),
+                'traceback': traceback.format_exc()
+            })
+        }
+
+async def crawl(url, browser_config_dict, crawler_config_dict):
+    """
+    Run the crawler with the provided configurations, with Lambda-specific settings
+    """
+    # Start with user-provided config but override with Lambda-required settings
+    base_browser_config = BrowserConfig.load(browser_config_dict) if browser_config_dict else BrowserConfig()
+    
+    # Apply Lambda-specific browser configurations
+    browser_config = BrowserConfig(
+        verbose=True,
+        browser_type="chromium",
+        headless=True,
+        user_agent_mode="random",
+        light_mode=True,
+        use_managed_browser=False,
+        extra_args=[
+            "--headless=new",
+            "--no-sandbox",
+            "--disable-dev-shm-usage",
+            "--disable-setuid-sandbox",
+            "--remote-allow-origins=*",
+            "--autoplay-policy=user-gesture-required",
+            "--single-process",            
+        ],
+        # # Carry over any other settings from user config that aren't overridden
+        # **{k: v for k, v in base_browser_config.model_dump().items() 
+        #    if k not in ['verbose', 'browser_type', 'headless', 'user_agent_mode', 
+        #                'light_mode', 'use_managed_browser', 'extra_args']}
+    )
+    
+    # Start with user-provided crawler config but ensure cache is bypassed
+    base_crawler_config = CrawlerRunConfig.load(crawler_config_dict) if crawler_config_dict else CrawlerRunConfig()
+    
+    # Apply Lambda-specific crawler configurations
+    crawler_config = CrawlerRunConfig(
+        exclude_external_links=base_crawler_config.exclude_external_links,
+        remove_overlay_elements=True,
+        magic=True,
+        cache_mode=CacheMode.BYPASS,
+        # Carry over markdown generator and other settings
+        markdown_generator=base_crawler_config.markdown_generator
+    )
+    
+    # Perform the crawl with Lambda-optimized settings
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url=url, config=crawler_config)
+        
+        # Return serializable results
+        return result.model_dump()

deploy/modal/crawl4ai_api_service.py

@@ -0,0 +1,543 @@
+import os
+import time
+import uuid
+from datetime import datetime
+from typing import Dict, Any, Optional, List
+
+import modal
+from modal import Image, App, Volume, Secret, web_endpoint, function
+
+# Configuration
+APP_NAME = "crawl4ai-api"
+CRAWL4AI_VERSION = "next"  # Using the 'next' branch
+PYTHON_VERSION = "3.10"  # Compatible with playwright
+DEFAULT_CREDITS = 1000
+
+# Create a custom image with Crawl4ai and its dependencies
+image = Image.debian_slim(python_version=PYTHON_VERSION).pip_install(
+    ["fastapi[standard]", "pymongo", "pydantic"]
+).run_commands(
+    "apt-get update",
+    "apt-get install -y software-properties-common",
+    "apt-get install -y git",
+    "apt-add-repository non-free",
+    "apt-add-repository contrib",
+    # Install crawl4ai from the next branch
+    f"pip install -U git+https://github.com/unclecode/crawl4ai.git@{CRAWL4AI_VERSION}",
+    "pip install -U fastapi[standard]",
+    "pip install -U pydantic",
+    # Install playwright and browsers
+    "crawl4ai-setup",
+)
+
+# Create persistent volume for user database
+user_db = Volume.from_name("crawl4ai-users", create_if_missing=True)
+
+# Create admin secret for secure operations
+admin_secret = Secret.from_name("admin-secret", create_if_missing=True)
+
+# Define the app
+app = App(APP_NAME, image=image)
+
+# Default configurations
+DEFAULT_BROWSER_CONFIG = {
+    "headless": True,
+    "verbose": False,
+}
+
+DEFAULT_CRAWLER_CONFIG = {
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "markdown_generator": {
+                "type": "DefaultMarkdownGenerator",
+                "params": {
+                    "content_filter": {
+                        "type": "PruningContentFilter",
+                        "params": {
+                            "threshold": 0.48,
+                            "threshold_type": "fixed"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+# Database operations
+@app.function(volumes={"/data": user_db})
+def init_db() -> None:
+    """Initialize database with indexes."""
+    from pymongo import MongoClient, ASCENDING
+    
+    client = MongoClient("mongodb://localhost:27017")
+    db = client.crawl4ai_db
+    
+    # Ensure indexes for faster lookups
+    db.users.create_index([("api_token", ASCENDING)], unique=True)
+    db.users.create_index([("email", ASCENDING)], unique=True)
+    
+    # Create usage stats collection
+    db.usage_stats.create_index([("user_id", ASCENDING), ("timestamp", ASCENDING)])
+    
+    print("Database initialized with required indexes")
+
+@app.function(volumes={"/data": user_db})
+def get_user_by_token(api_token: str) -> Optional[Dict[str, Any]]:
+    """Get user by API token."""
+    from pymongo import MongoClient
+    from bson.objectid import ObjectId
+    
+    client = MongoClient("mongodb://localhost:27017")
+    db = client.crawl4ai_db
+    
+    user = db.users.find_one({"api_token": api_token})
+    if not user:
+        return None
+    
+    # Convert ObjectId to string for serialization
+    user["_id"] = str(user["_id"])
+    return user
+
+@app.function(volumes={"/data": user_db})
+def create_user(email: str, name: str) -> Dict[str, Any]:
+    """Create a new user with initial credits."""
+    from pymongo import MongoClient
+    from bson.objectid import ObjectId
+    
+    client = MongoClient("mongodb://localhost:27017")
+    db = client.crawl4ai_db
+    
+    # Generate API token
+    api_token = str(uuid.uuid4())
+    
+    user = {
+        "email": email,
+        "name": name,
+        "api_token": api_token,
+        "credits": DEFAULT_CREDITS,
+        "created_at": datetime.utcnow(),
+        "updated_at": datetime.utcnow(),
+        "is_active": True
+    }
+    
+    try:
+        result = db.users.insert_one(user)
+        user["_id"] = str(result.inserted_id)
+        return user
+    except Exception as e:
+        if "duplicate key error" in str(e):
+            return {"error": "User with this email already exists"}
+        raise
+
+@app.function(volumes={"/data": user_db})
+def update_user_credits(api_token: str, amount: int) -> Dict[str, Any]:
+    """Update user credits (add or subtract)."""
+    from pymongo import MongoClient
+    
+    client = MongoClient("mongodb://localhost:27017")
+    db = client.crawl4ai_db
+    
+    # First get current user to check credits
+    user = db.users.find_one({"api_token": api_token})
+    if not user:
+        return {"success": False, "error": "User not found"}
+    
+    # For deductions, ensure sufficient credits
+    if amount < 0 and user["credits"] + amount < 0:
+        return {"success": False, "error": "Insufficient credits"}
+    
+    # Update credits
+    result = db.users.update_one(
+        {"api_token": api_token},
+        {
+            "$inc": {"credits": amount},
+            "$set": {"updated_at": datetime.utcnow()}
+        }
+    )
+    
+    if result.modified_count == 1:
+        # Get updated user
+        updated_user = db.users.find_one({"api_token": api_token})
+        return {
+            "success": True, 
+            "credits": updated_user["credits"]
+        }
+    else:
+        return {"success": False, "error": "Failed to update credits"}
+
+@app.function(volumes={"/data": user_db})
+def log_usage(user_id: str, url: str, success: bool, error: Optional[str] = None) -> None:
+    """Log usage statistics."""
+    from pymongo import MongoClient
+    from bson.objectid import ObjectId
+    
+    client = MongoClient("mongodb://localhost:27017")
+    db = client.crawl4ai_db
+    
+    log_entry = {
+        "user_id": user_id,
+        "url": url,
+        "timestamp": datetime.utcnow(),
+        "success": success,
+        "error": error
+    }
+    
+    db.usage_stats.insert_one(log_entry)
+
+# Main crawling function
+@app.function(timeout=300)  # 5 minute timeout
+async def crawl(
+    url: str,
+    browser_config: Optional[Dict[str, Any]] = None,
+    crawler_config: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """
+    Crawl a given URL using Crawl4ai.
+    
+    Args:
+        url: The URL to crawl
+        browser_config: Optional browser configuration to override defaults
+        crawler_config: Optional crawler configuration to override defaults
+        
+    Returns:
+        A dictionary containing the crawl results
+    """
+    from crawl4ai import (
+        AsyncWebCrawler,
+        BrowserConfig,
+        CrawlerRunConfig,
+        CrawlResult
+    )
+
+    # Prepare browser config using the loader method
+    if browser_config is None:
+        browser_config = DEFAULT_BROWSER_CONFIG
+    browser_config_obj = BrowserConfig.load(browser_config)
+    
+    # Prepare crawler config using the loader method
+    if crawler_config is None:
+        crawler_config = DEFAULT_CRAWLER_CONFIG
+    crawler_config_obj = CrawlerRunConfig.load(crawler_config)    
+    
+    # Perform the crawl
+    async with AsyncWebCrawler(config=browser_config_obj) as crawler:
+        result: CrawlResult = await crawler.arun(url=url, config=crawler_config_obj)
+        
+        # Return serializable results
+        try:
+            # Try newer Pydantic v2 method
+            return result.model_dump()
+        except AttributeError:
+            try:
+                # Try older Pydantic v1 method
+                return result.dict()
+            except AttributeError:
+                # Fallback to manual conversion
+                return {
+                    "url": result.url,
+                    "title": result.title,
+                    "status": result.status,
+                    "content": str(result.content) if hasattr(result, "content") else None,
+                    "links": [{"url": link.url, "text": link.text} for link in result.links] if hasattr(result, "links") else [],
+                    "markdown_v2": {
+                        "raw_markdown": result.markdown_v2.raw_markdown if hasattr(result, "markdown_v2") else None
+                    }
+                }
+
+# API endpoints
+@app.function()
+@web_endpoint(method="POST")
+def crawl_endpoint(data: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Web endpoint that accepts POST requests with JSON data containing:
+    - api_token: User's API token
+    - url: The URL to crawl
+    - browser_config: Optional browser configuration
+    - crawler_config: Optional crawler configuration
+    
+    Returns the crawl results and remaining credits.
+    """
+    # Extract and validate API token
+    api_token = data.get("api_token")
+    if not api_token:
+        return {
+            "success": False,
+            "error": "API token is required",
+            "status_code": 401
+        }
+    
+    # Verify user
+    user = get_user_by_token.remote(api_token)
+    if not user:
+        return {
+            "success": False,
+            "error": "Invalid API token",
+            "status_code": 401
+        }
+    
+    if not user.get("is_active", False):
+        return {
+            "success": False,
+            "error": "Account is inactive",
+            "status_code": 403
+        }
+    
+    # Validate URL
+    url = data.get("url")
+    if not url:
+        return {
+            "success": False,
+            "error": "URL is required",
+            "status_code": 400
+        }
+    
+    # Check credits
+    if user.get("credits", 0) <= 0:
+        return {
+            "success": False,
+            "error": "Insufficient credits",
+            "status_code": 403
+        }
+    
+    # Deduct credit first (1 credit per call)
+    credit_result = update_user_credits.remote(api_token, -1)
+    if not credit_result.get("success", False):
+        return {
+            "success": False,
+            "error": credit_result.get("error", "Failed to process credits"),
+            "status_code": 500
+        }
+    
+    # Extract configs
+    browser_config = data.get("browser_config")
+    crawler_config = data.get("crawler_config")
+    
+    # Perform crawl
+    try:
+        start_time = time.time()
+        result = crawl.remote(url, browser_config, crawler_config)
+        execution_time = time.time() - start_time
+        
+        # Log successful usage
+        log_usage.spawn(user["_id"], url, True)
+        
+        return {
+            "success": True,
+            "data": result,
+            "credits_remaining": credit_result.get("credits"),
+            "execution_time_seconds": round(execution_time, 2),
+            "status_code": 200
+        }
+    except Exception as e:
+        # Log failed usage
+        log_usage.spawn(user["_id"], url, False, str(e))
+        
+        # Return error
+        return {
+            "success": False,
+            "error": f"Crawling error: {str(e)}",
+            "credits_remaining": credit_result.get("credits"),
+            "status_code": 500
+        }
+
+# Admin endpoints
+@app.function(secrets=[admin_secret])
+@web_endpoint(method="POST")
+def admin_create_user(data: Dict[str, Any]) -> Dict[str, Any]:
+    """Admin endpoint to create new users."""
+    # Validate admin token
+    admin_token = data.get("admin_token")
+    if admin_token != os.environ.get("ADMIN_TOKEN"):
+        return {
+            "success": False,
+            "error": "Invalid admin token",
+            "status_code": 401
+        }
+    
+    # Validate input
+    email = data.get("email")
+    name = data.get("name")
+    
+    if not email or not name:
+        return {
+            "success": False,
+            "error": "Email and name are required",
+            "status_code": 400
+        }
+    
+    # Create user
+    user = create_user.remote(email, name)
+    
+    if "error" in user:
+        return {
+            "success": False,
+            "error": user["error"],
+            "status_code": 400
+        }
+    
+    return {
+        "success": True,
+        "data": {
+            "user_id": user["_id"],
+            "email": user["email"],
+            "name": user["name"],
+            "api_token": user["api_token"],
+            "credits": user["credits"],
+            "created_at": user["created_at"].isoformat() if isinstance(user["created_at"], datetime) else user["created_at"]
+        },
+        "status_code": 201
+    }
+
+@app.function(secrets=[admin_secret])
+@web_endpoint(method="POST")
+def admin_update_credits(data: Dict[str, Any]) -> Dict[str, Any]:
+    """Admin endpoint to update user credits."""
+    # Validate admin token
+    admin_token = data.get("admin_token")
+    if admin_token != os.environ.get("ADMIN_TOKEN"):
+        return {
+            "success": False,
+            "error": "Invalid admin token",
+            "status_code": 401
+        }
+    
+    # Validate input
+    api_token = data.get("api_token")
+    amount = data.get("amount")
+    
+    if not api_token:
+        return {
+            "success": False,
+            "error": "API token is required",
+            "status_code": 400
+        }
+    
+    if not isinstance(amount, int):
+        return {
+            "success": False,
+            "error": "Amount must be an integer",
+            "status_code": 400
+        }
+    
+    # Update credits
+    result = update_user_credits.remote(api_token, amount)
+    
+    if not result.get("success", False):
+        return {
+            "success": False,
+            "error": result.get("error", "Failed to update credits"),
+            "status_code": 400
+        }
+    
+    return {
+        "success": True,
+        "data": {
+            "credits": result["credits"]
+        },
+        "status_code": 200
+    }
+
+@app.function(secrets=[admin_secret])
+@web_endpoint(method="GET")
+def admin_get_users(admin_token: str) -> Dict[str, Any]:
+    """Admin endpoint to list all users."""
+    # Validate admin token
+    if admin_token != os.environ.get("ADMIN_TOKEN"):
+        return {
+            "success": False,
+            "error": "Invalid admin token",
+            "status_code": 401
+        }
+    
+    users = get_all_users.remote()
+    
+    return {
+        "success": True,
+        "data": users,
+        "status_code": 200
+    }
+
+@app.function(volumes={"/data": user_db})
+def get_all_users() -> List[Dict[str, Any]]:
+    """Get all users (for admin)."""
+    from pymongo import MongoClient
+    
+    client = MongoClient("mongodb://localhost:27017")
+    db = client.crawl4ai_db
+    
+    users = []
+    for user in db.users.find():
+        # Convert ObjectId to string
+        user["_id"] = str(user["_id"])
+        # Convert datetime to ISO format
+        for field in ["created_at", "updated_at"]:
+            if field in user and isinstance(user[field], datetime):
+                user[field] = user[field].isoformat()
+        users.append(user)
+    
+    return users
+
+# Public endpoints
+@app.function()
+@web_endpoint(method="GET")
+def health_check() -> Dict[str, Any]:
+    """Health check endpoint."""
+    return {
+        "status": "online",
+        "service": APP_NAME,
+        "version": CRAWL4AI_VERSION,
+        "timestamp": datetime.utcnow().isoformat()
+    }
+
+@app.function()
+@web_endpoint(method="GET")
+def check_credits(api_token: str) -> Dict[str, Any]:
+    """Check user credits."""
+    if not api_token:
+        return {
+            "success": False,
+            "error": "API token is required",
+            "status_code": 401
+        }
+    
+    user = get_user_by_token.remote(api_token)
+    if not user:
+        return {
+            "success": False,
+            "error": "Invalid API token",
+            "status_code": 401
+        }
+    
+    return {
+        "success": True,
+        "data": {
+            "credits": user["credits"],
+            "email": user["email"],
+            "name": user["name"]
+        },
+        "status_code": 200
+    }
+
+# Local entrypoint for testing
+@app.local_entrypoint()
+def main(url: str = "https://www.modal.com"):
+    """Command line entrypoint for local testing."""
+    print("Initializing database...")
+    init_db.remote()
+    
+    print(f"Testing crawl on URL: {url}")
+    result = crawl.remote(url)
+    
+    # Print sample of result
+    print("\nCrawl Result Sample:")
+    if "title" in result:
+        print(f"Title: {result['title']}")
+    if "status" in result:
+        print(f"Status: {result['status']}")
+    if "links" in result:
+        print(f"Links found: {len(result['links'])}")
+    if "markdown_v2" in result and result["markdown_v2"] and "raw_markdown" in result["markdown_v2"]:
+        print("\nMarkdown Preview (first 300 chars):")
+        print(result["markdown_v2"]["raw_markdown"][:300] + "...")

deploy/modal/entry.py

@@ -0,0 +1,127 @@
+import modal
+from typing import Optional, Dict, Any
+
+# Create a custom image with Crawl4ai and its dependencies
+# "pip install crawl4ai",
+image = modal.Image.debian_slim(python_version="3.10").pip_install(["fastapi[standard]"]).run_commands(
+    "apt-get update",
+    "apt-get install -y software-properties-common",
+    "apt-get install -y git",
+    "apt-add-repository non-free",
+    "apt-add-repository contrib",
+    "pip install -U git+https://github.com/unclecode/crawl4ai.git@next",
+    "pip install -U fastapi[standard]",
+    "pip install -U pydantic",
+    "crawl4ai-setup",  # This installs playwright and downloads chromium
+    # Print fastpi version
+    "python -m fastapi --version",
+)
+
+# Define the app
+app = modal.App("crawl4ai", image=image)
+
+# Define default configurations
+DEFAULT_BROWSER_CONFIG = {
+    "headless": True,
+    "verbose": False,
+}
+
+DEFAULT_CRAWLER_CONFIG = {
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "markdown_generator": {
+                "type": "DefaultMarkdownGenerator",
+                "params": {
+                    "content_filter": {
+                        "type": "PruningContentFilter",
+                        "params": {
+                            "threshold": 0.48,
+                            "threshold_type": "fixed"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+@app.function(timeout=300)  # 5 minute timeout
+async def crawl(
+    url: str,
+    browser_config: Optional[Dict[str, Any]] = None,
+    crawler_config: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """
+    Crawl a given URL using Crawl4ai.
+    
+    Args:
+        url: The URL to crawl
+        browser_config: Optional browser configuration to override defaults
+        crawler_config: Optional crawler configuration to override defaults
+        
+    Returns:
+        A dictionary containing the crawl results
+    """
+    from crawl4ai import (
+        AsyncWebCrawler,
+        BrowserConfig,
+        CrawlerRunConfig,
+        CrawlResult
+    )
+
+
+    # Prepare browser config using the loader method
+    if browser_config is None:
+        browser_config = DEFAULT_BROWSER_CONFIG
+    browser_config_obj = BrowserConfig.load(browser_config)
+    
+    # Prepare crawler config using the loader method
+    if crawler_config is None:
+        crawler_config = DEFAULT_CRAWLER_CONFIG
+    crawler_config_obj = CrawlerRunConfig.load(crawler_config)    
+    
+    
+    # Perform the crawl
+    async with AsyncWebCrawler(config=browser_config_obj) as crawler:
+        result: CrawlResult = await crawler.arun(url=url, config=crawler_config_obj)
+        
+        # Return serializable results
+        try:
+            # Try newer Pydantic v2 method
+            return result.model_dump()
+        except AttributeError:
+            try:
+                # Try older Pydantic v1 method
+                return result.__dict__
+            except AttributeError:
+                # Fallback to returning the raw result
+                return result
+
+@app.function()
+@modal.web_endpoint(method="POST")
+def crawl_endpoint(data: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Web endpoint that accepts POST requests with JSON data containing:
+    - url: The URL to crawl
+    - browser_config: Optional browser configuration
+    - crawler_config: Optional crawler configuration
+    
+    Returns the crawl results.
+    """
+    url = data.get("url")
+    if not url:
+        return {"error": "URL is required"}
+    
+    browser_config = data.get("browser_config")
+    crawler_config = data.get("crawler_config")
+    
+    return crawl.remote(url, browser_config, crawler_config)
+
+@app.local_entrypoint()
+def main(url: str = "https://www.modal.com"):
+    """
+    Command line entrypoint for local testing.
+    """
+    result = crawl.remote(url)
+    print(result)

deploy/modal/guide.md

@@ -0,0 +1,453 @@
+# Deploying Crawl4ai with Modal: A Comprehensive Tutorial
+
+Hey there! UncleCode here. I'm excited to show you how to deploy Crawl4ai using Modal - a fantastic serverless platform that makes deployment super simple and scalable.
+
+In this tutorial, I'll walk you through deploying your own Crawl4ai instance on Modal's infrastructure. This will give you a powerful, scalable web crawling solution without having to worry about infrastructure management.
+
+## What is Modal?
+
+Modal is a serverless platform that allows you to run Python functions in the cloud without managing servers. It's perfect for deploying Crawl4ai because:
+
+1. It handles all the infrastructure for you
+2. It scales automatically based on demand
+3. It makes deployment incredibly simple
+
+## Prerequisites
+
+Before we get started, you'll need:
+
+- A Modal account (sign up at [modal.com](https://modal.com))
+- Python 3.10 or later installed on your local machine
+- Basic familiarity with Python and command-line operations
+
+## Step 1: Setting Up Your Modal Account
+
+First, sign up for a Modal account at [modal.com](https://modal.com) if you haven't already. Modal offers a generous free tier that's perfect for getting started.
+
+After signing up, install the Modal CLI and authenticate:
+
+```bash
+pip install modal
+modal token new
+```
+
+This will open a browser window where you can authenticate and generate a token for the CLI.
+
+## Step 2: Creating Your Crawl4ai Deployment
+
+Now, let's create a Python file called `crawl4ai_modal.py` with our deployment code:
+
+```python
+import modal
+from typing import Optional, Dict, Any
+
+# Create a custom image with Crawl4ai and its dependencies
+image = modal.Image.debian_slim(python_version="3.10").pip_install(
+    ["fastapi[standard]"]
+).run_commands(
+    "apt-get update",
+    "apt-get install -y software-properties-common",
+    "apt-get install -y git",
+    "apt-add-repository non-free",
+    "apt-add-repository contrib",
+    "pip install -U crawl4ai",
+    "pip install -U fastapi[standard]",
+    "pip install -U pydantic",
+    "crawl4ai-setup",  # This installs playwright and downloads chromium
+)
+
+# Define the app
+app = modal.App("crawl4ai", image=image)
+
+# Define default configurations
+DEFAULT_BROWSER_CONFIG = {
+    "headless": True,
+    "verbose": False,
+}
+
+DEFAULT_CRAWLER_CONFIG = {
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "markdown_generator": {
+                "type": "DefaultMarkdownGenerator",
+                "params": {
+                    "content_filter": {
+                        "type": "PruningContentFilter",
+                        "params": {
+                            "threshold": 0.48,
+                            "threshold_type": "fixed"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+@app.function(timeout=300)  # 5 minute timeout
+async def crawl(
+    url: str,
+    browser_config: Optional[Dict[str, Any]] = None,
+    crawler_config: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """
+    Crawl a given URL using Crawl4ai.
+    
+    Args:
+        url: The URL to crawl
+        browser_config: Optional browser configuration to override defaults
+        crawler_config: Optional crawler configuration to override defaults
+        
+    Returns:
+        A dictionary containing the crawl results
+    """
+    from crawl4ai import (
+        AsyncWebCrawler,
+        BrowserConfig,
+        CrawlerRunConfig,
+        CrawlResult
+    )
+
+    # Prepare browser config using the loader method
+    if browser_config is None:
+        browser_config = DEFAULT_BROWSER_CONFIG
+    browser_config_obj = BrowserConfig.load(browser_config)
+    
+    # Prepare crawler config using the loader method
+    if crawler_config is None:
+        crawler_config = DEFAULT_CRAWLER_CONFIG
+    crawler_config_obj = CrawlerRunConfig.load(crawler_config)    
+    
+    # Perform the crawl
+    async with AsyncWebCrawler(config=browser_config_obj) as crawler:
+        result: CrawlResult = await crawler.arun(url=url, config=crawler_config_obj)
+        
+        # Return serializable results
+        try:
+            # Try newer Pydantic v2 method
+            return result.model_dump()
+        except AttributeError:
+            try:
+                # Try older Pydantic v1 method
+                return result.dict()
+            except AttributeError:
+                # Fallback to manual conversion
+                return {
+                    "url": result.url,
+                    "title": result.title,
+                    "status": result.status,
+                    "content": str(result.content) if hasattr(result, "content") else None,
+                    "links": [{"url": link.url, "text": link.text} for link in result.links] if hasattr(result, "links") else [],
+                    "markdown_v2": {
+                        "raw_markdown": result.markdown_v2.raw_markdown if hasattr(result, "markdown_v2") else None
+                    }
+                }
+
+@app.function()
+@modal.web_endpoint(method="POST")
+def crawl_endpoint(data: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Web endpoint that accepts POST requests with JSON data containing:
+    - url: The URL to crawl
+    - browser_config: Optional browser configuration
+    - crawler_config: Optional crawler configuration
+    
+    Returns the crawl results.
+    """
+    url = data.get("url")
+    if not url:
+        return {"error": "URL is required"}
+    
+    browser_config = data.get("browser_config")
+    crawler_config = data.get("crawler_config")
+    
+    return crawl.remote(url, browser_config, crawler_config)
+
+@app.local_entrypoint()
+def main(url: str = "https://www.modal.com"):
+    """
+    Command line entrypoint for local testing.
+    """
+    result = crawl.remote(url)
+    print(result)
+```
+
+## Step 3: Understanding the Code Components
+
+Let's break down what's happening in this code:
+
+### 1. Image Definition
+
+```python
+image = modal.Image.debian_slim(python_version="3.10").pip_install(
+    ["fastapi[standard]"]
+).run_commands(
+    "apt-get update",
+    "apt-get install -y software-properties-common",
+    "apt-get install -y git",
+    "apt-add-repository non-free",
+    "apt-add-repository contrib",
+    "pip install -U git+https://github.com/unclecode/crawl4ai.git@next",
+    "pip install -U fastapi[standard]",
+    "pip install -U pydantic",
+    "crawl4ai-setup",  # This installs playwright and downloads chromium
+)
+```
+
+This section defines the container image that Modal will use to run your code. It:
+- Starts with a Debian Slim base image with Python 3.10
+- Installs FastAPI
+- Updates the system packages
+- Installs Git and other dependencies
+- Installs Crawl4ai from the GitHub repository
+- Runs the Crawl4ai setup to install Playwright and download Chromium
+
+### 2. Modal App Definition
+
+```python
+app = modal.App("crawl4ai", image=image)
+```
+
+This creates a Modal application named "crawl4ai" that uses the image we defined above.
+
+### 3. Default Configurations
+
+```python
+DEFAULT_BROWSER_CONFIG = {
+    "headless": True,
+    "verbose": False,
+}
+
+DEFAULT_CRAWLER_CONFIG = {
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "markdown_generator": {
+                "type": "DefaultMarkdownGenerator",
+                "params": {
+                    "content_filter": {
+                        "type": "PruningContentFilter",
+                        "params": {
+                            "threshold": 0.48,
+                            "threshold_type": "fixed"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+```
+
+These define the default configurations for the browser and crawler. You can customize these settings based on your specific needs.
+
+### 4. The Crawl Function
+
+```python
+@app.function(timeout=300)
+async def crawl(url, browser_config, crawler_config):
+    # Function implementation
+```
+
+This is the main function that performs the crawling. It:
+- Takes a URL and optional configurations
+- Sets up the browser and crawler with those configurations
+- Performs the crawl
+- Returns the results in a serializable format
+
+The `@app.function(timeout=300)` decorator tells Modal to run this function in the cloud with a 5-minute timeout.
+
+### 5. The Web Endpoint
+
+```python
+@app.function()
+@modal.web_endpoint(method="POST")
+def crawl_endpoint(data: Dict[str, Any]) -> Dict[str, Any]:
+    # Function implementation
+```
+
+This creates a web endpoint that accepts POST requests. It:
+- Extracts the URL and configurations from the request
+- Calls the crawl function with those parameters
+- Returns the results
+
+### 6. Local Entrypoint
+
+```python
+@app.local_entrypoint()
+def main(url: str = "https://www.modal.com"):
+    # Function implementation
+```
+
+This provides a way to test the application from the command line.
+
+## Step 4: Testing Locally
+
+Before deploying, let's test our application locally:
+
+```bash
+modal run crawl4ai_modal.py --url "https://example.com"
+```
+
+This command will:
+1. Upload your code to Modal
+2. Create the necessary containers
+3. Run the `main` function with the specified URL
+4. Return the results
+
+Modal will handle all the infrastructure setup for you. You should see the crawling results printed to your console.
+
+## Step 5: Deploying Your Application
+
+Once you're satisfied with the local testing, it's time to deploy:
+
+```bash
+modal deploy crawl4ai_modal.py
+```
+
+This will deploy your application to Modal's cloud. The deployment process will output URLs for your web endpoints.
+
+You should see output similar to:
+
+```
+✓ Deployed crawl4ai.
+  URLs:
+    crawl_endpoint => https://your-username--crawl-endpoint.modal.run
+```
+
+Save this URL - you'll need it to make requests to your deployment.
+
+## Step 6: Using Your Deployment
+
+Now that your application is deployed, you can use it by sending POST requests to the endpoint URL:
+
+```bash
+curl -X POST https://your-username--crawl-endpoint.modal.run \
+  -H "Content-Type: application/json" \
+  -d '{"url": "https://example.com"}'
+```
+
+Or in Python:
+
+```python
+import requests
+
+response = requests.post(
+    "https://your-username--crawl-endpoint.modal.run",
+    json={"url": "https://example.com"}
+)
+
+result = response.json()
+print(result)
+```
+
+You can also customize the browser and crawler configurations:
+
+```python
+requests.post(
+    "https://your-username--crawl-endpoint.modal.run",
+    json={
+        "url": "https://example.com",
+        "browser_config": {
+            "headless": False,
+            "verbose": True
+        },
+        "crawler_config": {
+            "crawler_config": {
+                "type": "CrawlerRunConfig",
+                "params": {
+                    "markdown_generator": {
+                        "type": "DefaultMarkdownGenerator",
+                        "params": {
+                            "content_filter": {
+                                "type": "PruningContentFilter",
+                                "params": {
+                                    "threshold": 0.6,  # Adjusted threshold
+                                    "threshold_type": "fixed"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+)
+```
+
+## Step 7: Calling Your Deployment from Another Python Script
+
+You can also call your deployed function directly from another Python script:
+
+```python
+import modal
+
+# Get a reference to the deployed function
+crawl_function = modal.Function.from_name("crawl4ai", "crawl")
+
+# Call the function
+result = crawl_function.remote("https://example.com")
+print(result)
+```
+
+## Understanding Modal's Execution Flow
+
+To understand how Modal works, it's important to know:
+
+1. **Local vs. Remote Execution**: When you call a function with `.remote()`, it runs in Modal's cloud, not on your local machine.
+
+2. **Container Lifecycle**: Modal creates containers on-demand and destroys them when they're not needed.
+
+3. **Caching**: Modal caches your container images to speed up subsequent runs.
+
+4. **Serverless Scaling**: Modal automatically scales your application based on demand.
+
+## Customizing Your Deployment
+
+You can customize your deployment in several ways:
+
+### Changing the Crawl4ai Version
+
+To use a different version of Crawl4ai, update the installation command in the image definition:
+
+```python
+"pip install -U git+https://github.com/unclecode/crawl4ai.git@main",  # Use main branch
+```
+
+### Adjusting Resource Limits
+
+You can change the resources allocated to your functions:
+
+```python
+@app.function(timeout=600, cpu=2, memory=4096)  # 10 minute timeout, 2 CPUs, 4GB RAM
+async def crawl(...):
+    # Function implementation
+```
+
+### Keeping Containers Warm
+
+To reduce cold start times, you can keep containers warm:
+
+```python
+@app.function(keep_warm=1)  # Keep 1 container warm
+async def crawl(...):
+    # Function implementation
+```
+
+## Conclusion
+
+That's it! You've successfully deployed Crawl4ai on Modal. You now have a scalable web crawling solution that can handle as many requests as you need without requiring any infrastructure management.
+
+The beauty of this setup is its simplicity - Modal handles all the hard parts, letting you focus on using Crawl4ai to extract the data you need.
+
+Feel free to reach out if you have any questions or need help with your deployment!
+
+Happy crawling!
+- UncleCode
+
+## Additional Resources
+
+- [Modal Documentation](https://modal.com/docs)
+- [Crawl4ai GitHub Repository](https://github.com/unclecode/crawl4ai)
+- [Crawl4ai Documentation](https://docs.crawl4ai.com)

deploy/modal/test_modal.py

@@ -0,0 +1,317 @@
+
+#!/usr/bin/env python3
+"""
+Crawl4ai API Testing Script
+
+This script tests all endpoints of the Crawl4ai API service and demonstrates their usage.
+"""
+
+import argparse
+import json
+import sys
+import time
+from typing import Dict, Any, List, Optional
+
+import requests
+
+# Colors for terminal output
+class Colors:
+    HEADER = '\033[95m'
+    BLUE = '\033[94m'
+    GREEN = '\033[92m'
+    YELLOW = '\033[93m'
+    RED = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    UNDERLINE = '\033[4m'
+
+def print_header(text: str) -> None:
+    """Print a formatted header."""
+    print(f"\n{Colors.HEADER}{Colors.BOLD}{'=' * 80}{Colors.ENDC}")
+    print(f"{Colors.HEADER}{Colors.BOLD}{text.center(80)}{Colors.ENDC}")
+    print(f"{Colors.HEADER}{Colors.BOLD}{'=' * 80}{Colors.ENDC}\n")
+
+def print_step(text: str) -> None:
+    """Print a formatted step description."""
+    print(f"{Colors.BLUE}{Colors.BOLD}>> {text}{Colors.ENDC}")
+
+def print_success(text: str) -> None:
+    """Print a success message."""
+    print(f"{Colors.GREEN}✓ {text}{Colors.ENDC}")
+
+def print_warning(text: str) -> None:
+    """Print a warning message."""
+    print(f"{Colors.YELLOW}⚠ {text}{Colors.ENDC}")
+
+def print_error(text: str) -> None:
+    """Print an error message."""
+    print(f"{Colors.RED}✗ {text}{Colors.ENDC}")
+
+def print_json(data: Dict[str, Any]) -> None:
+    """Pretty print JSON data."""
+    print(json.dumps(data, indent=2))
+
+def make_request(method: str, url: str, params: Optional[Dict[str, Any]] = None, 
+                 json_data: Optional[Dict[str, Any]] = None, 
+                 expected_status: int = 200) -> Dict[str, Any]:
+    """Make an HTTP request and handle errors."""
+    print_step(f"Making {method.upper()} request to {url}")
+    
+    if params:
+        print(f"  Parameters: {params}")
+    if json_data:
+        print(f"  JSON Data: {json_data}")
+    
+    try:
+        response = requests.request(
+            method=method,
+            url=url,
+            params=params,
+            json=json_data,
+            timeout=300  # 5 minute timeout for crawling operations
+        )
+        
+        status_code = response.status_code
+        print(f"  Status Code: {status_code}")
+        
+        try:
+            data = response.json()
+            print("  Response:")
+            print_json(data)
+            
+            if status_code != expected_status:
+                print_error(f"Expected status code {expected_status}, got {status_code}")
+                return data
+            
+            print_success("Request successful")
+            return data
+        except ValueError:
+            print_error("Response is not valid JSON")
+            print(response.text)
+            return {"error": "Invalid JSON response"}
+            
+    except requests.RequestException as e:
+        print_error(f"Request failed: {str(e)}")
+        return {"error": str(e)}
+
+def test_health_check(base_url: str) -> bool:
+    """Test the health check endpoint."""
+    print_header("Testing Health Check Endpoint")
+    
+    response = make_request("GET", f"{base_url}/health_check")
+    
+    if "status" in response and response["status"] == "online":
+        print_success("Health check passed")
+        return True
+    else:
+        print_error("Health check failed")
+        return False
+
+def test_admin_create_user(base_url: str, admin_token: str, email: str, name: str) -> Optional[str]:
+    """Test creating a new user."""
+    print_header("Testing Admin User Creation")
+    
+    response = make_request(
+        "POST", 
+        f"{base_url}/admin_create_user",
+        json_data={
+            "admin_token": admin_token,
+            "email": email,
+            "name": name
+        },
+        expected_status=201
+    )
+    
+    if response.get("success") and "data" in response:
+        api_token = response["data"].get("api_token")
+        if api_token:
+            print_success(f"User created successfully with API token: {api_token}")
+            return api_token
+    
+    print_error("Failed to create user")
+    return None
+
+def test_check_credits(base_url: str, api_token: str) -> Optional[int]:
+    """Test checking user credits."""
+    print_header("Testing Check Credits Endpoint")
+    
+    response = make_request(
+        "GET",
+        f"{base_url}/check_credits",
+        params={"api_token": api_token}
+    )
+    
+    if response.get("success") and "data" in response:
+        credits = response["data"].get("credits")
+        if credits is not None:
+            print_success(f"User has {credits} credits")
+            return credits
+    
+    print_error("Failed to check credits")
+    return None
+
+def test_crawl_endpoint(base_url: str, api_token: str, url: str) -> bool:
+    """Test the crawl endpoint."""
+    print_header("Testing Crawl Endpoint")
+    
+    response = make_request(
+        "POST",
+        f"{base_url}/crawl_endpoint",
+        json_data={
+            "api_token": api_token,
+            "url": url
+        }
+    )
+    
+    if response.get("success") and "data" in response:
+        print_success("Crawl completed successfully")
+        
+        # Display some crawl result data
+        data = response["data"]
+        if "title" in data:
+            print(f"Page Title: {data['title']}")
+        if "status" in data:
+            print(f"Status: {data['status']}")
+        if "links" in data:
+            print(f"Links found: {len(data['links'])}")
+        if "markdown_v2" in data and data["markdown_v2"] and "raw_markdown" in data["markdown_v2"]:
+            print("Markdown Preview (first 200 chars):")
+            print(data["markdown_v2"]["raw_markdown"][:200] + "...")
+            
+        credits_remaining = response.get("credits_remaining")
+        if credits_remaining is not None:
+            print(f"Credits remaining: {credits_remaining}")
+            
+        return True
+    
+    print_error("Crawl failed")
+    return False
+
+def test_admin_update_credits(base_url: str, admin_token: str, api_token: str, amount: int) -> bool:
+    """Test updating user credits."""
+    print_header("Testing Admin Update Credits")
+    
+    response = make_request(
+        "POST",
+        f"{base_url}/admin_update_credits",
+        json_data={
+            "admin_token": admin_token,
+            "api_token": api_token,
+            "amount": amount
+        }
+    )
+    
+    if response.get("success") and "data" in response:
+        print_success(f"Credits updated successfully, new balance: {response['data'].get('credits')}")
+        return True
+    
+    print_error("Failed to update credits")
+    return False
+
+def test_admin_get_users(base_url: str, admin_token: str) -> List[Dict[str, Any]]:
+    """Test getting all users."""
+    print_header("Testing Admin Get All Users")
+    
+    response = make_request(
+        "GET",
+        f"{base_url}/admin_get_users",
+        params={"admin_token": admin_token}
+    )
+    
+    if response.get("success") and "data" in response:
+        users = response["data"]
+        print_success(f"Retrieved {len(users)} users")
+        return users
+    
+    print_error("Failed to get users")
+    return []
+
+def run_full_test(base_url: str, admin_token: str) -> None:
+    """Run all tests in sequence."""
+    # Remove trailing slash if present
+    base_url = base_url.rstrip('/')
+    
+    # Test 1: Health Check
+    if not test_health_check(base_url):
+        print_error("Health check failed, aborting tests")
+        sys.exit(1)
+    
+    # Test 2: Create a test user
+    email = f"test-user-{int(time.time())}@example.com"
+    name = "Test User"
+    api_token = test_admin_create_user(base_url, admin_token, email, name)
+    
+    if not api_token:
+        print_error("User creation failed, aborting tests")
+        sys.exit(1)
+    
+    # Test 3: Check initial credits
+    initial_credits = test_check_credits(base_url, api_token)
+    
+    if initial_credits is None:
+        print_error("Credit check failed, aborting tests")
+        sys.exit(1)
+    
+    # Test 4: Perform a crawl
+    test_url = "https://news.ycombinator.com"
+    crawl_success = test_crawl_endpoint(base_url, api_token, test_url)
+    
+    if not crawl_success:
+        print_warning("Crawl test failed, but continuing with other tests")
+    
+    # Test 5: Check credits after crawl
+    post_crawl_credits = test_check_credits(base_url, api_token)
+    
+    if post_crawl_credits is not None and initial_credits is not None:
+        if post_crawl_credits == initial_credits - 1:
+            print_success("Credit deduction verified")
+        else:
+            print_warning(f"Unexpected credit change: {initial_credits} -> {post_crawl_credits}")
+    
+    # Test 6: Add credits
+    add_credits_amount = 50
+    if test_admin_update_credits(base_url, admin_token, api_token, add_credits_amount):
+        print_success(f"Added {add_credits_amount} credits")
+    
+    # Test 7: Check credits after addition
+    post_addition_credits = test_check_credits(base_url, api_token)
+    
+    if post_addition_credits is not None and post_crawl_credits is not None:
+        if post_addition_credits == post_crawl_credits + add_credits_amount:
+            print_success("Credit addition verified")
+        else:
+            print_warning(f"Unexpected credit change: {post_crawl_credits} -> {post_addition_credits}")
+    
+    # Test 8: Get all users
+    users = test_admin_get_users(base_url, admin_token)
+    
+    if users:
+        # Check if our test user is in the list
+        test_user = next((user for user in users if user.get("email") == email), None)
+        if test_user:
+            print_success("Test user found in users list")
+        else:
+            print_warning("Test user not found in users list")
+    
+    # Final report
+    print_header("Test Summary")
+    
+    print_success("All endpoints tested successfully")
+    print(f"Test user created with email: {email}")
+    print(f"API token: {api_token}")
+    print(f"Final credit balance: {post_addition_credits}")
+
+def main():
+    parser = argparse.ArgumentParser(description="Test Crawl4ai API endpoints")
+    parser.add_argument("--base-url", required=True, help="Base URL of the Crawl4ai API (e.g., https://username--crawl4ai-api.modal.run)")
+    parser.add_argument("--admin-token", required=True, help="Admin token for authentication")
+    
+    args = parser.parse_args()
+    
+    print_header("Crawl4ai API Test Script")
+    print(f"Testing API at: {args.base_url}")
+    
+    run_full_test(args.base_url, args.admin_token)
+
+if __name__ == "__main__":
+    main()

docs/examples/crypto_analysis_example.py

@@ -0,0 +1,230 @@
+"""
+Crawl4AI Crypto Trading Analysis Demo
+Author: Unclecode
+Date: 2024-03-15
+
+This script demonstrates advanced crypto market analysis using:
+1. Web scraping of real-time CoinMarketCap data
+2. Smart table extraction with layout detection
+3. Hedge fund-grade financial metrics
+4. Interactive visualizations for trading signals
+
+Key Features:
+- Volume Anomaly Detection: Finds unusual trading activity
+- Liquidity Power Score: Identifies easily tradable assets
+- Volatility-Weighted Momentum: Surface sustainable trends
+- Smart Money Signals: Algorithmic buy/hold recommendations
+"""
+
+import asyncio
+import pandas as pd
+import plotly.express as px
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LXMLWebScrapingStrategy
+from crawl4ai import CrawlResult
+from typing import List
+from IPython.display import HTML
+
+class CryptoAlphaGenerator:
+    """
+    Advanced crypto analysis engine that transforms raw web data into:
+    - Volume anomaly flags
+    - Liquidity scores
+    - Momentum-risk ratios
+    - Machine learning-inspired trading signals
+    
+    Methods:
+    analyze_tables(): Process raw tables into trading insights
+    create_visuals(): Generate institutional-grade visualizations
+    generate_insights(): Create plain English trading recommendations
+    """
+    
+    def clean_data(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Convert crypto market data to machine-readable format
+        Handles currency symbols, units (B=Billions), and percentage values
+        """
+        # Clean numeric columns
+        df['Price'] = df['Price'].str.replace('[^\d.]', '', regex=True).astype(float)
+        df['Market Cap'] = df['Market Cap'].str.extract(r'\$([\d.]+)B')[0].astype(float) * 1e9
+        df['Volume(24h)'] = df['Volume(24h)'].str.extract(r'\$([\d.]+)B')[0].astype(float) * 1e9
+        
+        # Convert percentages to decimal values
+        for col in ['1h %', '24h %', '7d %']:
+            df[col] = df[col].str.replace('%', '').astype(float) / 100
+            
+        return df
+
+    def calculate_metrics(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Compute advanced trading metrics used by quantitative funds:
+        
+        1. Volume/Market Cap Ratio - Measures liquidity efficiency
+           (High ratio = Underestimated attention)
+           
+        2. Volatility Score - Risk-adjusted momentum potential
+           (STD of 1h/24h/7d returns)
+           
+        3. Momentum Score - Weighted average of returns
+           (1h:30% + 24h:50% + 7d:20%)
+           
+        4. Volume Anomaly - 3σ deviation detection
+           (Flags potential insider activity)
+        """
+        # Liquidity Metrics
+        df['Volume/Market Cap Ratio'] = df['Volume(24h)'] / df['Market Cap']
+        
+        # Risk Metrics
+        df['Volatility Score'] = df[['1h %','24h %','7d %']].std(axis=1)
+        
+        # Momentum Metrics
+        df['Momentum Score'] = (df['1h %']*0.3 + df['24h %']*0.5 + df['7d %']*0.2)
+        
+        # Anomaly Detection
+        median_vol = df['Volume(24h)'].median()
+        df['Volume Anomaly'] = df['Volume(24h)'] > 3 * median_vol
+        
+        # Value Flags
+        df['Undervalued Flag'] = (df['Market Cap'] < 1e9) & (df['Momentum Score'] > 0.05)
+        df['Liquid Giant'] = (df['Volume/Market Cap Ratio'] > 0.15) & (df['Market Cap'] > 1e9)
+        
+        return df
+
+    def create_visuals(self, df: pd.DataFrame) -> dict:
+        """
+        Generate three institutional-grade visualizations:
+        
+        1. 3D Market Map - X:Size, Y:Liquidity, Z:Momentum
+        2. Liquidity Tree - Color:Volume Efficiency
+        3. Momentum Leaderboard - Top sustainable movers
+        """
+        # 3D Market Overview
+        fig1 = px.scatter_3d(
+            df, 
+            x='Market Cap', 
+            y='Volume/Market Cap Ratio',
+            z='Momentum Score',
+            size='Volatility Score',
+            color='Volume Anomaly',
+            hover_name='Name',
+            title='Smart Money Market Map: Spot Overlooked Opportunities',
+            labels={'Market Cap': 'Size (Log $)', 'Volume/Market Cap Ratio': 'Liquidity Power'},
+            log_x=True,
+            template='plotly_dark'
+        )
+        
+        # Liquidity Efficiency Tree
+        fig2 = px.treemap(
+            df,
+            path=['Name'], 
+            values='Market Cap',
+            color='Volume/Market Cap Ratio',
+            hover_data=['Momentum Score'],
+            title='Liquidity Forest: Green = High Trading Efficiency',
+            color_continuous_scale='RdYlGn'
+        )
+        
+        # Momentum Leaders
+        fig3 = px.bar(
+            df.sort_values('Momentum Score', ascending=False).head(10),
+            x='Name', 
+            y='Momentum Score',
+            color='Volatility Score',
+            title='Sustainable Momentum Leaders (Low Volatility + High Growth)',
+            text='7d %',
+            template='plotly_dark'
+        )
+        
+        return {'market_map': fig1, 'liquidity_tree': fig2, 'momentum_leaders': fig3}
+
+    def generate_insights(self, df: pd.DataFrame) -> str:
+        """
+        Create plain English trading insights explaining:
+        - Volume spikes and their implications
+        - Risk-reward ratios of top movers
+        - Liquidity warnings for large positions
+        """
+        top_coin = df.sort_values('Momentum Score', ascending=False).iloc[0]
+        anomaly_coins = df[df['Volume Anomaly']].sort_values('Volume(24h)', ascending=False)
+        
+        report = f"""
+        🚀 Top Alpha Opportunity: {top_coin['Name']}
+        - Momentum Score: {top_coin['Momentum Score']:.2%} (Top 1%)
+        - Risk-Reward Ratio: {top_coin['Momentum Score']/top_coin['Volatility Score']:.1f}
+        - Liquidity Warning: {'✅ Safe' if top_coin['Liquid Giant'] else '⚠️ Thin Markets'}
+        
+        🔥 Volume Spikes Detected ({len(anomaly_coins)} coins):
+        {anomaly_coins[['Name', 'Volume(24h)']].head(3).to_markdown(index=False)}
+        
+        💡 Smart Money Tip: Coins with Volume/Cap > 15% and Momentum > 5% 
+        historically outperform by 22% weekly returns.
+        """
+        return report
+
+async def main():
+    """
+    Main execution flow:
+    1. Configure headless browser for scraping
+    2. Extract live crypto market data
+    3. Clean and analyze using hedge fund models
+    4. Generate visualizations and insights
+    5. Output professional trading report
+    """
+    # Configure browser with anti-detection features
+    browser_config = BrowserConfig(
+        headless=True,
+        stealth=True,
+        block_resources=["image", "media"]
+    )
+    
+    # Initialize crawler with smart table detection
+    crawler = AsyncWebCrawler(config=browser_config)
+    await crawler.start()
+    
+    try:
+        # Set up scraping parameters
+        crawl_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            scraping_strategy=LXMLWebScrapingStrategy(
+                table_score_threshold=8,  # Strict table detection
+                keep_data_attributes=True
+            )
+        )
+        
+        # Execute market data extraction
+        results: List[CrawlResult] = await crawler.arun(
+            url='https://coinmarketcap.com/?page=1',
+            config=crawl_config
+        )
+        
+        # Process results
+        for result in results:
+            if result.success and result.media['tables']:
+                # Extract primary market table
+                raw_df = pd.DataFrame(
+                    result.media['tables'][0]['rows'],
+                    columns=result.media['tables'][0]['headers']
+                )
+                
+                # Initialize analysis engine
+                analyzer = CryptoAlphaGenerator()
+                clean_df = analyzer.clean_data(raw_df)
+                analyzed_df = analyzer.calculate_metrics(clean_df)
+                
+                # Generate outputs
+                visuals = analyzer.create_visuals(analyzed_df)
+                insights = analyzer.generate_insights(analyzed_df)
+                
+                # Save visualizations
+                visuals['market_map'].write_html("market_map.html")
+                visuals['liquidity_tree'].write_html("liquidity_tree.html")
+                
+                # Display results
+                print("🔑 Key Trading Insights:")
+                print(insights)
+                print("\n📊 Open 'market_map.html' for interactive analysis")
+
+    finally:
+        await crawler.close()
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/dispatcher_example.py

@@ -39,7 +39,7 @@ async def memory_adaptive_with_rate_limit(urls, browser_config, run_config):
     start = time.perf_counter()
     async with AsyncWebCrawler(config=browser_config) as crawler:
         dispatcher = MemoryAdaptiveDispatcher(
-            memory_threshold_percent=70.0,
+            memory_threshold_percent=95.0,
             max_session_permit=10,
             rate_limiter=RateLimiter(
                 base_delay=(1.0, 2.0), max_delay=30.0, max_retries=2

docs/examples/extraction_strategies_examples.py

@@ -11,7 +11,7 @@ import asyncio
 import os
 
 from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 from crawl4ai.extraction_strategy import (
     LLMExtractionStrategy,
     JsonCssExtractionStrategy,

docs/examples/llm_extraction_openai_pricing.py

@@ -1,4 +1,4 @@
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 from crawl4ai import AsyncWebCrawler, LLMExtractionStrategy
 import asyncio
 import os

docs/examples/llm_markdown_generator.py

@@ -1,7 +1,7 @@
 import os
 import asyncio
 from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 from crawl4ai.content_filter_strategy import LLMContentFilter
 
 async def test_llm_filter():

docs/examples/quickstart_async.config.py

@@ -1,6 +1,6 @@
 import os, sys
 
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 
 sys.path.append(
     os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

docs/examples/quickstart_async.py

@@ -1,6 +1,6 @@
 import os, sys
 
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 
 # append parent directory to system path
 sys.path.append(

docs/examples/quickstart_sync.py

@@ -1,6 +1,6 @@
 import os
 import time
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 from crawl4ai.web_crawler import WebCrawler
 from crawl4ai.chunking_strategy import *
 from crawl4ai.extraction_strategy import *

docs/examples/tutorial_v0.5.py

@@ -13,11 +13,11 @@ from crawl4ai.deep_crawling import (
 )
 from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer
 from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy
-from crawl4ai.configs import ProxyConfig
+from crawl4ai.proxy_strategy import ProxyConfig
 from crawl4ai import RoundRobinProxyStrategy
 from crawl4ai.content_filter_strategy import LLMContentFilter
 from crawl4ai import DefaultMarkdownGenerator
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
 from crawl4ai.processors.pdf import PDFCrawlerStrategy, PDFContentScrapingStrategy
 from pprint import pprint

docs/md_v2/api/parameters.md

@@ -71,7 +71,8 @@ We group them by category.
 | **`word_count_threshold`**   | `int` (default: ~200)                | Skips text blocks below X words. Helps ignore trivial sections.                                 |
 | **`extraction_strategy`**    | `ExtractionStrategy` (default: None) | If set, extracts structured data (CSS-based, LLM-based, etc.).                                  |
 | **`markdown_generator`**     | `MarkdownGenerationStrategy` (None)  | If you want specialized markdown output (citations, filtering, chunking, etc.).                 |
-| **`css_selector`**           | `str` (None)                         | Retains only the part of the page matching this selector.                                       |
+| **`css_selector`**           | `str` (None)                         | Retains only the part of the page matching this selector. Affects the entire extraction process. |
+| **`target_elements`**        | `List[str]` (None)                   | List of CSS selectors for elements to focus on for markdown generation and data extraction, while still processing the entire page for links, media, etc. Provides more flexibility than `css_selector`. |
 | **`excluded_tags`**          | `list` (None)                        | Removes entire tags (e.g. `["script", "style"]`).                                               |
 | **`excluded_selector`**      | `str` (None)                         | Like `css_selector` but to exclude. E.g. `"#ads, .tracker"`.                                    |
 | **`only_text`**              | `bool` (False)                       | If `True`, tries to extract text-only content.                                                  |

docs/md_v2/api/strategies.md

@@ -131,7 +131,7 @@ OverlappingWindowChunking(
 ```python
 from pydantic import BaseModel
 from crawl4ai.extraction_strategy import LLMExtractionStrategy
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 
 # Define schema
 class Article(BaseModel):
@@ -198,7 +198,7 @@ result = await crawler.arun(
 
 ```python
 from crawl4ai.chunking_strategy import OverlappingWindowChunking
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 
 # Create chunking strategy
 chunker = OverlappingWindowChunking(

docs/md_v2/blog/releases/0.5.0.md

@@ -251,7 +251,7 @@ from crawl4ai import (
     RoundRobinProxyStrategy,
 )
 import asyncio
-from crawl4ai.configs import ProxyConfig
+from crawl4ai.proxy_strategy import ProxyConfig
 async def main():
     # Load proxies and create rotation strategy
     proxies = ProxyConfig.from_env()
@@ -305,7 +305,7 @@ asyncio.run(main())
 ```python
 from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, DefaultMarkdownGenerator
 from crawl4ai.content_filter_strategy import LLMContentFilter
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 import asyncio
 
 llm_config = LLMConfig(provider="gemini/gemini-1.5-pro", api_token="env:GEMINI_API_KEY")
@@ -335,7 +335,7 @@ asyncio.run(main())
 
 ```python
 from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 
 llm_config = LLMConfig(provider="gemini/gemini-1.5-pro", api_token="env:GEMINI_API_KEY")
 
@@ -401,7 +401,7 @@ print(schema)
   experimentation between different LLM configurations.
 
   ```python
-  from crawl4ai.types import LLMConfig
+  from crawl4ai import LLMConfig
   from crawl4ai.extraction_strategy import LLMExtractionStrategy
   from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
 

docs/md_v2/core/content-selection.md

@@ -8,6 +8,10 @@ Below, we show how to configure these parameters and combine them for precise co
 
 ## 1. CSS-Based Selection
 
+There are two ways to select content from a page: using `css_selector` or the more flexible `target_elements`.
+
+### 1.1 Using `css_selector`
+
 A straightforward way to **limit** your crawl results to a certain region of the page is **`css_selector`** in **`CrawlerRunConfig`**:
 
 ```python
@@ -32,6 +36,33 @@ if __name__ == "__main__":
 
 **Result**: Only elements matching that selector remain in `result.cleaned_html`.
 
+### 1.2 Using `target_elements`
+
+The `target_elements` parameter provides more flexibility by allowing you to target **multiple elements** for content extraction while preserving the entire page context for other features:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def main():
+    config = CrawlerRunConfig(
+        # Target article body and sidebar, but not other content
+        target_elements=["article.main-content", "aside.sidebar"]
+    )
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://example.com/blog-post", 
+            config=config
+        )
+        print("Markdown focused on target elements")
+        print("Links from entire page still available:", len(result.links.get("internal", [])))
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+**Key difference**: With `target_elements`, the markdown generation and structural data extraction focus on those elements, but other page elements (like links, images, and tables) are still extracted from the entire page. This gives you fine-grained control over what appears in your markdown content while preserving full page context for link analysis and media collection.
+
 ---
 
 ## 2. Content Filtering & Exclusions
@@ -404,15 +435,59 @@ Stick to BeautifulSoup strategy (default) when:
 
 ---
 
-## 7. Conclusion
+## 7. Combining CSS Selection Methods
 
-By mixing **css_selector** scoping, **content filtering** parameters, and advanced **extraction strategies**, you can precisely **choose** which data to keep. Key parameters in **`CrawlerRunConfig`** for content selection include:
+You can combine `css_selector` and `target_elements` in powerful ways to achieve fine-grained control over your output:
 
-1. **`css_selector`** – Basic scoping to an element or region.  
-2. **`word_count_threshold`** – Skip short blocks.  
-3. **`excluded_tags`** – Remove entire HTML tags.  
-4. **`exclude_external_links`**, **`exclude_social_media_links`**, **`exclude_domains`** – Filter out unwanted links or domains.  
-5. **`exclude_external_images`** – Remove images from external sources.  
-6. **`process_iframes`** – Merge iframe content if needed.  
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
+
+async def main():
+    # Target specific content but preserve page context
+    config = CrawlerRunConfig(
+        # Focus markdown on main content and sidebar
+        target_elements=["#main-content", ".sidebar"],
+        
+        # Global filters applied to entire page
+        excluded_tags=["nav", "footer", "header"],
+        exclude_external_links=True,
+        
+        # Use basic content thresholds
+        word_count_threshold=15,
+        
+        cache_mode=CacheMode.BYPASS
+    )
+    
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://example.com/article",
+            config=config
+        )
+        
+        print(f"Content focuses on specific elements, but all links still analyzed")
+        print(f"Internal links: {len(result.links.get('internal', []))}")
+        print(f"External links: {len(result.links.get('external', []))}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+This approach gives you the best of both worlds:
+- Markdown generation and content extraction focus on the elements you care about
+- Links, images and other page data still give you the full context of the page
+- Content filtering still applies globally
+
+## 8. Conclusion
+
+By mixing **target_elements** or **css_selector** scoping, **content filtering** parameters, and advanced **extraction strategies**, you can precisely **choose** which data to keep. Key parameters in **`CrawlerRunConfig`** for content selection include:
+
+1. **`target_elements`** – Array of CSS selectors to focus markdown generation and data extraction, while preserving full page context for links and media.
+2. **`css_selector`** – Basic scoping to an element or region for all extraction processes.  
+3. **`word_count_threshold`** – Skip short blocks.  
+4. **`excluded_tags`** – Remove entire HTML tags.  
+5. **`exclude_external_links`**, **`exclude_social_media_links`**, **`exclude_domains`** – Filter out unwanted links or domains.  
+6. **`exclude_external_images`** – Remove images from external sources.  
+7. **`process_iframes`** – Merge iframe content if needed.  
 
 Combine these with structured extraction (CSS, LLM-based, or others) to build powerful crawls that yield exactly the content you want, from raw or cleaned HTML up to sophisticated JSON structures. For more detail, see [Configuration Reference](../api/parameters.md). Enjoy curating your data to the max!

docs/md_v2/core/link-media.md

@@ -133,19 +133,28 @@ This approach is handy when you still want external links but need to block cert
 
 ### 3.1 Accessing `result.media`
 
-By default, Crawl4AI collects images, audio, and video URLs it finds on the page. These are stored in `result.media`, a dictionary keyed by media type (e.g., `images`, `videos`, `audio`).
+By default, Crawl4AI collects images, audio, video URLs, and data tables it finds on the page. These are stored in `result.media`, a dictionary keyed by media type (e.g., `images`, `videos`, `audio`, `tables`).
 
 **Basic Example**:
 
 ```python
 if result.success:
+    # Get images
     images_info = result.media.get("images", [])
     print(f"Found {len(images_info)} images in total.")
-    for i, img in enumerate(images_info[:5]):  # Inspect just the first 5
+    for i, img in enumerate(images_info[:3]):  # Inspect just the first 3
         print(f"[Image {i}] URL: {img['src']}")
         print(f"           Alt text: {img.get('alt', '')}")
         print(f"           Score: {img.get('score')}")
         print(f"           Description: {img.get('desc', '')}\n")
+    
+    # Get tables
+    tables = result.media.get("tables", [])
+    print(f"Found {len(tables)} data tables in total.")
+    for i, table in enumerate(tables):
+        print(f"[Table {i}] Caption: {table.get('caption', 'No caption')}")
+        print(f"           Columns: {len(table.get('headers', []))}")
+        print(f"           Rows: {len(table.get('rows', []))}")
 ```
 
 **Structure Example**:
@@ -171,6 +180,19 @@ result.media = {
   ],
   "audio": [
     # Similar structure but with audio-specific fields
+  ],
+  "tables": [
+    {
+      "headers": ["Name", "Age", "Location"],
+      "rows": [
+        ["John Doe", "34", "New York"],
+        ["Jane Smith", "28", "San Francisco"],
+        ["Alex Johnson", "42", "Chicago"]
+      ],
+      "caption": "Employee Directory",
+      "summary": "Directory of company employees"
+    },
+    # More tables if present
   ]
 }
 ```
@@ -199,7 +221,53 @@ crawler_cfg = CrawlerRunConfig(
 
 This setting attempts to discard images from outside the primary domain, keeping only those from the site you’re crawling.
 
-### 3.3 Additional Media Config
+### 3.3 Working with Tables
+
+Crawl4AI can detect and extract structured data from HTML tables. Tables are analyzed based on various criteria to determine if they are actual data tables (as opposed to layout tables), including:
+
+- Presence of thead and tbody sections
+- Use of th elements for headers
+- Column consistency
+- Text density
+- And other factors
+
+Tables that score above the threshold (default: 7) are extracted and stored in `result.media.tables`.
+
+**Accessing Table Data**:
+
+```python
+if result.success:
+    tables = result.media.get("tables", [])
+    print(f"Found {len(tables)} data tables on the page")
+    
+    if tables:
+        # Access the first table
+        first_table = tables[0]
+        print(f"Table caption: {first_table.get('caption', 'No caption')}")
+        print(f"Headers: {first_table.get('headers', [])}")
+        
+        # Print the first 3 rows
+        for i, row in enumerate(first_table.get('rows', [])[:3]):
+            print(f"Row {i+1}: {row}")
+```
+
+**Configuring Table Extraction**:
+
+You can adjust the sensitivity of the table detection algorithm with:
+
+```python
+crawler_cfg = CrawlerRunConfig(
+    table_score_threshold=5  # Lower value = more tables detected (default: 7)
+)
+```
+
+Each extracted table contains:
+- `headers`: Column header names
+- `rows`: List of rows, each containing cell values
+- `caption`: Table caption text (if available)
+- `summary`: Table summary attribute (if specified)
+
+### 3.4 Additional Media Config
 
 - **`screenshot`**: Set to `True` if you want a full-page screenshot stored as `base64` in `result.screenshot`.  
 - **`pdf`**: Set to `True` if you want a PDF version of the page in `result.pdf`.  
@@ -273,4 +341,11 @@ if __name__ == "__main__":
 
 ---
 
-**That’s it for Link & Media Analysis!** You’re now equipped to filter out unwanted sites and zero in on the images and videos that matter for your project.
+**That’s it for Link & Media Analysis!** You’re now equipped to filter out unwanted sites and zero in on the images and videos that matter for your project.
+### Table Extraction Tips
+
+- Not all HTML tables are extracted - only those detected as "data tables" vs. layout tables.
+- Tables with inconsistent cell counts, nested tables, or those used purely for layout may be skipped.
+- If you're missing tables, try adjusting the `table_score_threshold` to a lower value (default is 7).
+
+The table detection algorithm scores tables based on features like consistent columns, presence of headers, text density, and more. Tables scoring above the threshold are considered data tables worth extracting.

docs/md_v2/core/quickstart.md

@@ -128,7 +128,7 @@ Crawl4AI can also extract structured data (JSON) using CSS or XPath selectors. B
 
 ```python
 from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 
 # Generate a schema (one-time cost)
 html = "<div class='product'><h2>Gaming Laptop</h2><span class='price'>$999.99</span></div>"

docs/md_v2/extraction/no-llm-strategies.md

@@ -415,7 +415,7 @@ The schema generator is available as a static method on both `JsonCssExtractionS
 
 ```python
 from crawl4ai.extraction_strategy import JsonCssExtractionStrategy, JsonXPathExtractionStrategy
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 
 # Sample HTML with product information
 html = """

docs/snippets/deep_crawl/2.filters.py

@@ -0,0 +1,162 @@
+import asyncio
+from typing import List
+
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    BFSDeepCrawlStrategy,
+    CrawlResult,
+    URLFilter, # Base class for filters, not directly used in examples but good to import for context
+    ContentTypeFilter,
+    DomainFilter,
+    FilterChain,
+    URLPatternFilter,
+    SEOFilter # Advanced filter, can be introduced later or as bonus
+)
+
+async def deep_crawl_filter_tutorial_part_2():
+    """
+    Tutorial demonstrating URL filters in Crawl4AI, focusing on isolated filter behavior
+    before integrating them into a deep crawl.
+
+    This tutorial covers:
+    - Testing individual filters with synthetic URLs.
+    - Understanding filter logic and behavior in isolation.
+    - Combining filters using FilterChain.
+    - Integrating filters into a deep crawling example.
+    """
+
+    # === Introduction: URL Filters in Isolation ===
+    print("\n" + "=" * 40)
+    print("=== Introduction: URL Filters in Isolation ===")
+    print("=" * 40 + "\n")
+    print("In this section, we will explore each filter individually using synthetic URLs.")
+    print("This allows us to understand exactly how each filter works before using them in a crawl.\n")
+
+
+    # === 2. ContentTypeFilter - Testing in Isolation ===
+    print("\n" + "=" * 40)
+    print("=== 2. ContentTypeFilter - Testing in Isolation ===")
+    print("=" * 40 + "\n")
+
+    # 2.1. Create ContentTypeFilter:
+    # Create a ContentTypeFilter to allow only 'text/html' and 'application/json' content types 
+    # BASED ON URL EXTENSIONS.
+    content_type_filter = ContentTypeFilter(allowed_types=["text/html", "application/json"])
+    print("ContentTypeFilter created, allowing types (by extension): ['text/html', 'application/json']")
+    print("Note: ContentTypeFilter in Crawl4ai works by checking URL file extensions, not HTTP headers.")
+
+
+    # 2.2. Synthetic URLs for Testing:
+    # ContentTypeFilter checks URL extensions. We provide URLs with different extensions to test.
+    test_urls_content_type = [
+        "https://example.com/page.html",       # Should pass: .html extension (text/html)
+        "https://example.com/data.json",       # Should pass: .json extension (application/json)
+        "https://example.com/image.png",       # Should reject: .png extension (not allowed type)
+        "https://example.com/document.pdf",    # Should reject: .pdf extension (not allowed type)
+        "https://example.com/page",            # Should pass: no extension (defaults to allow) - check default behaviour!
+        "https://example.com/page.xhtml",      # Should pass: .xhtml extension (text/html)
+    ]
+
+    # 2.3. Apply Filter and Show Results:
+    print("\n=== Testing ContentTypeFilter (URL Extension based) ===")
+    for url in test_urls_content_type:
+        passed = content_type_filter.apply(url)
+        result = "PASSED" if passed else "REJECTED"
+        extension = ContentTypeFilter._extract_extension(url) # Show extracted extension for clarity
+        print(f"- URL: {url} - {result} (Extension: '{extension or 'No Extension'}')")
+    print("=" * 40)
+
+    input("Press Enter to continue to DomainFilter example...")
+
+    # === 3. DomainFilter - Testing in Isolation ===
+    print("\n" + "=" * 40)
+    print("=== 3. DomainFilter - Testing in Isolation ===")
+    print("=" * 40 + "\n")
+
+    # 3.1. Create DomainFilter:
+    domain_filter = DomainFilter(allowed_domains=["crawl4ai.com", "example.com"])
+    print("DomainFilter created, allowing domains: ['crawl4ai.com', 'example.com']")
+
+    # 3.2. Synthetic URLs for Testing:
+    test_urls_domain = [
+        "https://docs.crawl4ai.com/api",
+        "https://example.com/products",
+        "https://another-website.org/blog",
+        "https://sub.example.com/about",
+        "https://crawl4ai.com.attacker.net", # Corrected example: now should be rejected
+    ]
+
+    # 3.3. Apply Filter and Show Results:
+    print("\n=== Testing DomainFilter ===")
+    for url in test_urls_domain:
+        passed = domain_filter.apply(url)
+        result = "PASSED" if passed else "REJECTED"
+        print(f"- URL: {url} - {result}")
+    print("=" * 40)
+
+    input("Press Enter to continue to FilterChain example...")
+
+    # === 4. FilterChain - Combining Filters ===
+    print("\n" + "=" * 40)
+    print("=== 4. FilterChain - Combining Filters ===")
+    print("=" * 40 + "\n")
+
+    combined_filter = FilterChain(
+        filters=[
+            URLPatternFilter(patterns=["*api*"]),
+            ContentTypeFilter(allowed_types=["text/html"]), # Still URL extension based
+            DomainFilter(allowed_domains=["docs.crawl4ai.com"]),
+        ]
+    )
+    print("FilterChain created, combining URLPatternFilter, ContentTypeFilter, and DomainFilter.")
+
+
+    test_urls_combined = [
+        "https://docs.crawl4ai.com/api/async-webcrawler",
+        "https://example.com/api/products",
+        "https://docs.crawl4ai.com/core/crawling",
+        "https://another-website.org/api/data",
+    ]
+
+    # 4.3. Apply FilterChain and Show Results
+    print("\n=== Testing FilterChain (URLPatternFilter + ContentTypeFilter + DomainFilter) ===")
+    for url in test_urls_combined:
+        passed = await combined_filter.apply(url)
+        result = "PASSED" if passed else "REJECTED"
+        print(f"- URL: {url} - {result}")
+    print("=" * 40)
+
+    input("Press Enter to continue to Deep Crawl with FilterChain example...")
+
+    # === 5. Deep Crawl with FilterChain ===
+    print("\n" + "=" * 40)
+    print("=== 5. Deep Crawl with FilterChain ===")
+    print("=" * 40 + "\n")
+    print("Finally, let's integrate the FilterChain into a deep crawl example.")
+
+    config_final_crawl = CrawlerRunConfig(
+        deep_crawl_strategy=BFSDeepCrawlStrategy(
+            max_depth=2,
+            max_pages=10,
+            include_external=False,
+            filter_chain=combined_filter
+        ),
+        verbose=False,
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        results_final_crawl: List[CrawlResult] = await crawler.arun(
+            url="https://docs.crawl4ai.com", config=config_final_crawl
+        )
+
+        print("=== Crawled URLs (Deep Crawl with FilterChain) ===")
+        for result in results_final_crawl:
+            print(f"- {result.url}, Depth: {result.metadata.get('depth', 0)}")
+        print("=" * 40)
+
+    print("\nTutorial Completed! Review the output of each section to understand URL filters.")
+
+
+if __name__ == "__main__":
+    asyncio.run(deep_crawl_filter_tutorial_part_2())

tests/20241401/test_llm_filter.py

@@ -1,7 +1,7 @@
 import os
 import asyncio
 from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 from crawl4ai.content_filter_strategy import LLMContentFilter
 
 async def test_llm_filter():

tests/async/test_chunking_and_extraction_strategies.py

@@ -7,7 +7,7 @@ import json
 parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.append(parent_dir)
 
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 from crawl4ai.async_webcrawler import AsyncWebCrawler
 from crawl4ai.chunking_strategy import RegexChunking
 from crawl4ai.extraction_strategy import LLMExtractionStrategy

tests/browser/test_launch_standalone.py

@@ -0,0 +1,17 @@
+from crawl4ai.browser_profiler import BrowserProfiler
+import asyncio
+
+
+if __name__ == "__main__":
+    # Test launching a standalone browser
+    async def test_standalone_browser():
+        profiler = BrowserProfiler()
+        cdp_url = await profiler.launch_standalone_browser(
+            browser_type="chromium",
+            user_data_dir="~/.crawl4ai/browser_profile/test-browser-data",
+            debugging_port=9222,
+            headless=False
+        )
+        print(f"CDP URL: {cdp_url}")
+
+    asyncio.run(test_standalone_browser())

tests/docker/test_docker.py

@@ -7,7 +7,7 @@ from crawl4ai import (
     BrowserConfig, CrawlerRunConfig, DefaultMarkdownGenerator,
     PruningContentFilter, JsonCssExtractionStrategy, LLMContentFilter, CacheMode
 )
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 from crawl4ai.docker_client import Crawl4aiDockerClient
 
 class Crawl4AiTester:

tests/docker/test_serialization.py

@@ -2,7 +2,7 @@ import inspect
 from typing import Any, Dict
 from enum import Enum
 
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 
 def to_serializable_dict(obj: Any) -> Dict:
     """

tests/test_web_crawler.py

@@ -1,5 +1,5 @@
 import unittest, os
-from crawl4ai.types import LLMConfig
+from crawl4ai import LLMConfig
 from crawl4ai.web_crawler import WebCrawler
 from crawl4ai.chunking_strategy import (
     RegexChunking,