From 1e1c887a2f59dc2fcef1bd139ddee990ddc28ddd Mon Sep 17 00:00:00 2001 From: Emmanuel Ferdman Date: Tue, 13 May 2025 00:04:58 -0700 Subject: [PATCH 001/119] fix(docker-api): migrate to modern datetime library API Signed-off-by: Emmanuel Ferdman --- deploy/docker/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 732371f7..edfa51e5 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -4,7 +4,7 @@ import asyncio from typing import List, Tuple, Dict from functools import partial from uuid import uuid4 -from datetime import datetime +from datetime import datetime, timezone import logging from typing import Optional, AsyncGenerator @@ -542,7 +542,7 @@ async def handle_crawl_job( task_id = f"crawl_{uuid4().hex[:8]}" await redis.hset(f"task:{task_id}", mapping={ "status": TaskStatus.PROCESSING, # <-- keep enum values consistent - "created_at": datetime.utcnow().isoformat(), + "created_at": datetime.now(timezone.utc).replace(tzinfo=None).isoformat(), "url": json.dumps(urls), # store list as JSON string "result": "", "error": "", From 7a8190ecb67020743e1b1fb41c8b197e4507ed59 Mon Sep 17 00:00:00 2001 From: Nezar Ali Date: Wed, 6 Aug 2025 11:58:29 +0300 Subject: [PATCH 002/119] Fix examples in README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f70eb264..274e8075 100644 --- a/README.md +++ b/README.md @@ -347,7 +347,7 @@ async def main(): async with AsyncWebCrawler(config=browser_config) as crawler: result = await crawler.arun( - url="https://docs.micronaut.io/4.7.6/guide/", + url="https://docs.micronaut.io/4.9.9/guide/", config=run_config ) print(len(result.markdown.raw_markdown)) @@ -399,7 +399,7 @@ async def main(): "type": "attribute", "attribute": "src" } - } + ] } extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True) From be63c98db3513b6b54c91ef7d479936aa2966c4b Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 11 Aug 2025 13:25:17 +0800 Subject: [PATCH 003/119] feat(docker): add user-provided hooks support to Docker API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements comprehensive hooks functionality allowing users to provide custom Python functions as strings that execute at specific points in the crawling pipeline. Key Features: - Support for all 8 crawl4ai hook points: • on_browser_created: Initialize browser settings • on_page_context_created: Configure page context • before_goto: Pre-navigation setup • after_goto: Post-navigation processing • on_user_agent_updated: User agent modification handling • on_execution_started: Crawl execution initialization • before_retrieve_html: Pre-extraction processing • before_return_html: Final HTML processing Implementation Details: - Created UserHookManager for validation, compilation, and safe execution - Added IsolatedHookWrapper for error isolation and timeout protection - AST-based validation ensures code structure correctness - Sandboxed execution with restricted builtins for security - Configurable timeout (1-120 seconds) prevents infinite loops - Comprehensive error handling ensures hooks don't crash main process - Execution tracking with detailed statistics and logging API Changes: - Added HookConfig schema with code and timeout fields - Extended CrawlRequest with optional hooks parameter - Added /hooks/info endpoint for hook discovery - Updated /crawl and /crawl/stream endpoints to support hooks Safety Features: - Malformed hooks return clear validation errors - Hook errors are isolated and reported without stopping crawl - Execution statistics track success/failure/timeout rates - All hook results are JSON-serializable Testing: - Comprehensive test suite covering all 8 hooks - Error handling and timeout scenarios validated - Authentication, performance, and content extraction examples - 100% success rate in production testing Documentation: - Added extensive hooks section to docker-deployment.md - Security warnings about user-provided code risks - Real-world examples using httpbin.org, GitHub, BBC - Best practices and troubleshooting guide ref #1377 --- deploy/docker/api.py | 108 ++++- deploy/docker/hook_manager.py | 512 ++++++++++++++++++++++ deploy/docker/schemas.py | 44 ++ deploy/docker/server.py | 112 ++++- docs/examples/docker_hooks_examples.py | 513 +++++++++++++++++++++++ docs/md_v2/core/docker-deployment.md | 403 ++++++++++++++++++ tests/docker/test_hooks_client.py | 372 ++++++++++++++++ tests/docker/test_hooks_comprehensive.py | 512 ++++++++++++++++++++++ 8 files changed, 2555 insertions(+), 21 deletions(-) create mode 100644 deploy/docker/hook_manager.py create mode 100644 docs/examples/docker_hooks_examples.py create mode 100644 tests/docker/test_hooks_client.py create mode 100644 tests/docker/test_hooks_comprehensive.py diff --git a/deploy/docker/api.py b/deploy/docker/api.py index b54bae65..310077cf 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -419,13 +419,15 @@ async def handle_crawl_request( urls: List[str], browser_config: dict, crawler_config: dict, - config: dict + config: dict, + hooks_config: Optional[dict] = None ) -> dict: - """Handle non-streaming crawl requests.""" + """Handle non-streaming crawl requests with optional hooks.""" start_mem_mb = _get_memory_mb() # <--- Get memory before start_time = time.time() mem_delta_mb = None peak_mem_mb = start_mem_mb + hook_manager = None try: urls = [('https://' + url) if not url.startswith(('http://', 'https://')) else url for url in urls] @@ -445,6 +447,19 @@ async def handle_crawl_request( # crawler: AsyncWebCrawler = AsyncWebCrawler(config=browser_config) # await crawler.start() + # Attach hooks if provided + hooks_status = {} + if hooks_config: + from hook_manager import attach_user_hooks_to_crawler, UserHookManager + hook_manager = UserHookManager(timeout=hooks_config.get('timeout', 30)) + hooks_status, hook_manager = await attach_user_hooks_to_crawler( + crawler, + hooks_config.get('code', {}), + timeout=hooks_config.get('timeout', 30), + hook_manager=hook_manager + ) + logger.info(f"Hooks attachment status: {hooks_status['status']}") + base_config = config["crawler"]["base_config"] # Iterate on key-value pairs in global_config then use haseattr to set them for key, value in base_config.items(): @@ -458,6 +473,10 @@ async def handle_crawl_request( config=crawler_config, dispatcher=dispatcher) results = await partial_func() + + # Ensure results is always a list + if not isinstance(results, list): + results = [results] # await crawler.close() @@ -472,19 +491,68 @@ async def handle_crawl_request( # Process results to handle PDF bytes processed_results = [] for result in results: - result_dict = result.model_dump() - # If PDF exists, encode it to base64 - if result_dict.get('pdf') is not None: - result_dict['pdf'] = b64encode(result_dict['pdf']).decode('utf-8') - processed_results.append(result_dict) + try: + # Check if result has model_dump method (is a proper CrawlResult) + if hasattr(result, 'model_dump'): + result_dict = result.model_dump() + elif isinstance(result, dict): + result_dict = result + else: + # Handle unexpected result type + logger.warning(f"Unexpected result type: {type(result)}") + result_dict = { + "url": str(result) if hasattr(result, '__str__') else "unknown", + "success": False, + "error_message": f"Unexpected result type: {type(result).__name__}" + } + + # If PDF exists, encode it to base64 + if result_dict.get('pdf') is not None and isinstance(result_dict.get('pdf'), bytes): + result_dict['pdf'] = b64encode(result_dict['pdf']).decode('utf-8') + + processed_results.append(result_dict) + except Exception as e: + logger.error(f"Error processing result: {e}") + processed_results.append({ + "url": "unknown", + "success": False, + "error_message": str(e) + }) - return { + response = { "success": True, "results": processed_results, "server_processing_time_s": end_time - start_time, "server_memory_delta_mb": mem_delta_mb, "server_peak_memory_mb": peak_mem_mb } + + # Add hooks information if hooks were used + if hooks_config and hook_manager: + from hook_manager import UserHookManager + if isinstance(hook_manager, UserHookManager): + try: + # Ensure all hook data is JSON serializable + import json + hook_data = { + "status": hooks_status, + "execution_log": hook_manager.execution_log, + "errors": hook_manager.errors, + "summary": hook_manager.get_summary() + } + # Test that it's serializable + json.dumps(hook_data) + response["hooks"] = hook_data + except (TypeError, ValueError) as e: + logger.error(f"Hook data not JSON serializable: {e}") + response["hooks"] = { + "status": {"status": "error", "message": "Hook data serialization failed"}, + "execution_log": [], + "errors": [{"error": str(e)}], + "summary": {} + } + + return response except Exception as e: logger.error(f"Crawl error: {str(e)}", exc_info=True) @@ -513,9 +581,11 @@ async def handle_stream_crawl_request( urls: List[str], browser_config: dict, crawler_config: dict, - config: dict -) -> Tuple[AsyncWebCrawler, AsyncGenerator]: - """Handle streaming crawl requests.""" + config: dict, + hooks_config: Optional[dict] = None +) -> Tuple[AsyncWebCrawler, AsyncGenerator, Optional[Dict]]: + """Handle streaming crawl requests with optional hooks.""" + hooks_info = None try: browser_config = BrowserConfig.load(browser_config) # browser_config.verbose = True # Set to False or remove for production stress testing @@ -536,6 +606,20 @@ async def handle_stream_crawl_request( # crawler = AsyncWebCrawler(config=browser_config) # await crawler.start() + + # Attach hooks if provided + if hooks_config: + from hook_manager import attach_user_hooks_to_crawler, UserHookManager + hook_manager = UserHookManager(timeout=hooks_config.get('timeout', 30)) + hooks_status, hook_manager = await attach_user_hooks_to_crawler( + crawler, + hooks_config.get('code', {}), + timeout=hooks_config.get('timeout', 30), + hook_manager=hook_manager + ) + logger.info(f"Hooks attachment status for streaming: {hooks_status['status']}") + # Include hook manager in hooks_info for proper tracking + hooks_info = {'status': hooks_status, 'manager': hook_manager} results_gen = await crawler.arun_many( urls=urls, @@ -543,7 +627,7 @@ async def handle_stream_crawl_request( dispatcher=dispatcher ) - return crawler, results_gen + return crawler, results_gen, hooks_info except Exception as e: # Make sure to close crawler if started during an error here diff --git a/deploy/docker/hook_manager.py b/deploy/docker/hook_manager.py new file mode 100644 index 00000000..41c4f25d --- /dev/null +++ b/deploy/docker/hook_manager.py @@ -0,0 +1,512 @@ +""" +Hook Manager for User-Provided Hook Functions +Handles validation, compilation, and safe execution of user-provided hook code +""" + +import ast +import asyncio +import traceback +from typing import Dict, Callable, Optional, Tuple, List, Any +import logging + +logger = logging.getLogger(__name__) + + +class UserHookManager: + """Manages user-provided hook functions with error isolation""" + + # Expected signatures for each hook point + HOOK_SIGNATURES = { + "on_browser_created": ["browser"], + "on_page_context_created": ["page", "context"], + "before_goto": ["page", "context", "url"], + "after_goto": ["page", "context", "url", "response"], + "on_user_agent_updated": ["page", "context", "user_agent"], + "on_execution_started": ["page", "context"], + "before_retrieve_html": ["page", "context"], + "before_return_html": ["page", "context", "html"] + } + + # Default timeout for hook execution (in seconds) + DEFAULT_TIMEOUT = 30 + + def __init__(self, timeout: int = DEFAULT_TIMEOUT): + self.timeout = timeout + self.errors: List[Dict[str, Any]] = [] + self.compiled_hooks: Dict[str, Callable] = {} + self.execution_log: List[Dict[str, Any]] = [] + + def validate_hook_structure(self, hook_code: str, hook_point: str) -> Tuple[bool, str]: + """ + Validate the structure of user-provided hook code + + Args: + hook_code: The Python code string containing the hook function + hook_point: The hook point name (e.g., 'on_page_context_created') + + Returns: + Tuple of (is_valid, error_message) + """ + try: + # Parse the code + tree = ast.parse(hook_code) + + # Check if it's empty + if not tree.body: + return False, "Hook code is empty" + + # Find the function definition + func_def = None + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + func_def = node + break + + if not func_def: + return False, "Hook must contain a function definition (def or async def)" + + # Check if it's async (all hooks should be async) + if not isinstance(func_def, ast.AsyncFunctionDef): + return False, f"Hook function must be async (use 'async def' instead of 'def')" + + # Get function name for better error messages + func_name = func_def.name + + # Validate parameters + expected_params = self.HOOK_SIGNATURES.get(hook_point, []) + if not expected_params: + return False, f"Unknown hook point: {hook_point}" + + func_params = [arg.arg for arg in func_def.args.args] + + # Check if it has **kwargs for flexibility + has_kwargs = func_def.args.kwarg is not None + + # Must have at least the expected parameters + missing_params = [] + for expected in expected_params: + if expected not in func_params: + missing_params.append(expected) + + if missing_params and not has_kwargs: + return False, f"Hook function '{func_name}' must accept parameters: {', '.join(expected_params)} (missing: {', '.join(missing_params)})" + + # Check if it returns something (should return page or browser) + has_return = any(isinstance(node, ast.Return) for node in ast.walk(func_def)) + if not has_return: + # Warning, not error - we'll handle this + logger.warning(f"Hook function '{func_name}' should return the {expected_params[0]} object") + + return True, "Valid" + + except SyntaxError as e: + return False, f"Syntax error at line {e.lineno}: {str(e)}" + except Exception as e: + return False, f"Failed to parse hook code: {str(e)}" + + def compile_hook(self, hook_code: str, hook_point: str) -> Optional[Callable]: + """ + Compile user-provided hook code into a callable function + + Args: + hook_code: The Python code string + hook_point: The hook point name + + Returns: + Compiled function or None if compilation failed + """ + try: + # Create a safe namespace for the hook + # Use a more complete builtins that includes __import__ + import builtins + safe_builtins = {} + + # Add safe built-in functions + allowed_builtins = [ + 'print', 'len', 'str', 'int', 'float', 'bool', + 'list', 'dict', 'set', 'tuple', 'range', 'enumerate', + 'zip', 'map', 'filter', 'any', 'all', 'sum', 'min', 'max', + 'sorted', 'reversed', 'abs', 'round', 'isinstance', 'type', + 'getattr', 'hasattr', 'setattr', 'callable', 'iter', 'next', + '__import__', '__build_class__' # Required for exec + ] + + for name in allowed_builtins: + if hasattr(builtins, name): + safe_builtins[name] = getattr(builtins, name) + + namespace = { + '__name__': f'user_hook_{hook_point}', + '__builtins__': safe_builtins + } + + # Add commonly needed imports + exec("import asyncio", namespace) + exec("import json", namespace) + exec("import re", namespace) + exec("from typing import Dict, List, Optional", namespace) + + # Execute the code to define the function + exec(hook_code, namespace) + + # Find the async function in the namespace + for name, obj in namespace.items(): + if callable(obj) and not name.startswith('_') and asyncio.iscoroutinefunction(obj): + return obj + + # If no async function found, look for any function + for name, obj in namespace.items(): + if callable(obj) and not name.startswith('_'): + logger.warning(f"Found non-async function '{name}' - wrapping it") + # Wrap sync function in async + async def async_wrapper(*args, **kwargs): + return obj(*args, **kwargs) + return async_wrapper + + raise ValueError("No callable function found in hook code") + + except Exception as e: + error = { + 'hook_point': hook_point, + 'error': f"Failed to compile hook: {str(e)}", + 'type': 'compilation_error', + 'traceback': traceback.format_exc() + } + self.errors.append(error) + logger.error(f"Hook compilation failed for {hook_point}: {str(e)}") + return None + + async def execute_hook_safely( + self, + hook_func: Callable, + hook_point: str, + *args, + **kwargs + ) -> Tuple[Any, Optional[Dict]]: + """ + Execute a user hook with error isolation and timeout + + Args: + hook_func: The compiled hook function + hook_point: The hook point name + *args, **kwargs: Arguments to pass to the hook + + Returns: + Tuple of (result, error_dict) + """ + start_time = asyncio.get_event_loop().time() + + try: + # Add timeout to prevent infinite loops + result = await asyncio.wait_for( + hook_func(*args, **kwargs), + timeout=self.timeout + ) + + # Log successful execution + execution_time = asyncio.get_event_loop().time() - start_time + self.execution_log.append({ + 'hook_point': hook_point, + 'status': 'success', + 'execution_time': execution_time, + 'timestamp': start_time + }) + + return result, None + + except asyncio.TimeoutError: + error = { + 'hook_point': hook_point, + 'error': f'Hook execution timed out ({self.timeout}s limit)', + 'type': 'timeout', + 'execution_time': self.timeout + } + self.errors.append(error) + self.execution_log.append({ + 'hook_point': hook_point, + 'status': 'timeout', + 'error': error['error'], + 'execution_time': self.timeout, + 'timestamp': start_time + }) + # Return the first argument (usually page/browser) to continue + return args[0] if args else None, error + + except Exception as e: + execution_time = asyncio.get_event_loop().time() - start_time + error = { + 'hook_point': hook_point, + 'error': str(e), + 'type': type(e).__name__, + 'traceback': traceback.format_exc(), + 'execution_time': execution_time + } + self.errors.append(error) + self.execution_log.append({ + 'hook_point': hook_point, + 'status': 'failed', + 'error': str(e), + 'error_type': type(e).__name__, + 'execution_time': execution_time, + 'timestamp': start_time + }) + # Return the first argument (usually page/browser) to continue + return args[0] if args else None, error + + def get_summary(self) -> Dict[str, Any]: + """Get a summary of hook execution""" + total_hooks = len(self.execution_log) + successful = sum(1 for log in self.execution_log if log['status'] == 'success') + failed = sum(1 for log in self.execution_log if log['status'] == 'failed') + timed_out = sum(1 for log in self.execution_log if log['status'] == 'timeout') + + return { + 'total_executions': total_hooks, + 'successful': successful, + 'failed': failed, + 'timed_out': timed_out, + 'success_rate': (successful / total_hooks * 100) if total_hooks > 0 else 0, + 'total_errors': len(self.errors) + } + + +class IsolatedHookWrapper: + """Wraps user hooks with error isolation and reporting""" + + def __init__(self, hook_manager: UserHookManager): + self.hook_manager = hook_manager + + def create_hook_wrapper(self, user_hook: Callable, hook_point: str) -> Callable: + """ + Create a wrapper that isolates hook errors from main process + + Args: + user_hook: The compiled user hook function + hook_point: The hook point name + + Returns: + Wrapped async function that handles errors gracefully + """ + + async def wrapped_hook(*args, **kwargs): + """Wrapped hook with error isolation""" + # Get the main return object (page/browser) + # This ensures we always have something to return + return_obj = None + if args: + return_obj = args[0] + elif 'page' in kwargs: + return_obj = kwargs['page'] + elif 'browser' in kwargs: + return_obj = kwargs['browser'] + + try: + # Execute user hook with safety + result, error = await self.hook_manager.execute_hook_safely( + user_hook, + hook_point, + *args, + **kwargs + ) + + if error: + # Hook failed but we continue with original object + logger.warning(f"User hook failed at {hook_point}: {error['error']}") + return return_obj + + # Hook succeeded - return its result or the original object + if result is None: + logger.debug(f"Hook at {hook_point} returned None, using original object") + return return_obj + + return result + + except Exception as e: + # This should rarely happen due to execute_hook_safely + logger.error(f"Unexpected error in hook wrapper for {hook_point}: {e}") + return return_obj + + # Set function name for debugging + wrapped_hook.__name__ = f"wrapped_{hook_point}" + return wrapped_hook + + +async def process_user_hooks( + hooks_input: Dict[str, str], + timeout: int = 30 +) -> Tuple[Dict[str, Callable], List[Dict], UserHookManager]: + """ + Process and compile user-provided hook functions + + Args: + hooks_input: Dictionary mapping hook points to code strings + timeout: Timeout for each hook execution + + Returns: + Tuple of (compiled_hooks, validation_errors, hook_manager) + """ + + hook_manager = UserHookManager(timeout=timeout) + wrapper = IsolatedHookWrapper(hook_manager) + compiled_hooks = {} + validation_errors = [] + + for hook_point, hook_code in hooks_input.items(): + # Skip empty hooks + if not hook_code or not hook_code.strip(): + continue + + # Validate hook point + if hook_point not in UserHookManager.HOOK_SIGNATURES: + validation_errors.append({ + 'hook_point': hook_point, + 'error': f'Unknown hook point. Valid points: {", ".join(UserHookManager.HOOK_SIGNATURES.keys())}', + 'code_preview': hook_code[:100] + '...' if len(hook_code) > 100 else hook_code + }) + continue + + # Validate structure + is_valid, message = hook_manager.validate_hook_structure(hook_code, hook_point) + if not is_valid: + validation_errors.append({ + 'hook_point': hook_point, + 'error': message, + 'code_preview': hook_code[:100] + '...' if len(hook_code) > 100 else hook_code + }) + continue + + # Compile the hook + hook_func = hook_manager.compile_hook(hook_code, hook_point) + if hook_func: + # Wrap with error isolation + wrapped_hook = wrapper.create_hook_wrapper(hook_func, hook_point) + compiled_hooks[hook_point] = wrapped_hook + logger.info(f"Successfully compiled hook for {hook_point}") + else: + validation_errors.append({ + 'hook_point': hook_point, + 'error': 'Failed to compile hook function - check syntax and structure', + 'code_preview': hook_code[:100] + '...' if len(hook_code) > 100 else hook_code + }) + + return compiled_hooks, validation_errors, hook_manager + + +async def process_user_hooks_with_manager( + hooks_input: Dict[str, str], + hook_manager: UserHookManager +) -> Tuple[Dict[str, Callable], List[Dict]]: + """ + Process and compile user-provided hook functions with existing manager + + Args: + hooks_input: Dictionary mapping hook points to code strings + hook_manager: Existing UserHookManager instance + + Returns: + Tuple of (compiled_hooks, validation_errors) + """ + + wrapper = IsolatedHookWrapper(hook_manager) + compiled_hooks = {} + validation_errors = [] + + for hook_point, hook_code in hooks_input.items(): + # Skip empty hooks + if not hook_code or not hook_code.strip(): + continue + + # Validate hook point + if hook_point not in UserHookManager.HOOK_SIGNATURES: + validation_errors.append({ + 'hook_point': hook_point, + 'error': f'Unknown hook point. Valid points: {", ".join(UserHookManager.HOOK_SIGNATURES.keys())}', + 'code_preview': hook_code[:100] + '...' if len(hook_code) > 100 else hook_code + }) + continue + + # Validate structure + is_valid, message = hook_manager.validate_hook_structure(hook_code, hook_point) + if not is_valid: + validation_errors.append({ + 'hook_point': hook_point, + 'error': message, + 'code_preview': hook_code[:100] + '...' if len(hook_code) > 100 else hook_code + }) + continue + + # Compile the hook + hook_func = hook_manager.compile_hook(hook_code, hook_point) + if hook_func: + # Wrap with error isolation + wrapped_hook = wrapper.create_hook_wrapper(hook_func, hook_point) + compiled_hooks[hook_point] = wrapped_hook + logger.info(f"Successfully compiled hook for {hook_point}") + else: + validation_errors.append({ + 'hook_point': hook_point, + 'error': 'Failed to compile hook function - check syntax and structure', + 'code_preview': hook_code[:100] + '...' if len(hook_code) > 100 else hook_code + }) + + return compiled_hooks, validation_errors + + +async def attach_user_hooks_to_crawler( + crawler, # AsyncWebCrawler instance + user_hooks: Dict[str, str], + timeout: int = 30, + hook_manager: Optional[UserHookManager] = None +) -> Tuple[Dict[str, Any], UserHookManager]: + """ + Attach user-provided hooks to crawler with full error reporting + + Args: + crawler: AsyncWebCrawler instance + user_hooks: Dictionary mapping hook points to code strings + timeout: Timeout for each hook execution + hook_manager: Optional existing UserHookManager instance + + Returns: + Tuple of (status_dict, hook_manager) + """ + + # Use provided hook_manager or create a new one + if hook_manager is None: + hook_manager = UserHookManager(timeout=timeout) + + # Process hooks with the hook_manager + compiled_hooks, validation_errors = await process_user_hooks_with_manager( + user_hooks, hook_manager + ) + + # Log validation errors + if validation_errors: + logger.warning(f"Hook validation errors: {validation_errors}") + + # Attach successfully compiled hooks + attached_hooks = [] + for hook_point, wrapped_hook in compiled_hooks.items(): + try: + crawler.crawler_strategy.set_hook(hook_point, wrapped_hook) + attached_hooks.append(hook_point) + logger.info(f"Attached hook to {hook_point}") + except Exception as e: + logger.error(f"Failed to attach hook to {hook_point}: {e}") + validation_errors.append({ + 'hook_point': hook_point, + 'error': f'Failed to attach hook: {str(e)}' + }) + + status = 'success' if not validation_errors else ('partial' if attached_hooks else 'failed') + + status_dict = { + 'status': status, + 'attached_hooks': attached_hooks, + 'validation_errors': validation_errors, + 'total_hooks_provided': len(user_hooks), + 'successfully_attached': len(attached_hooks), + 'failed_validation': len(validation_errors) + } + + return status_dict, hook_manager \ No newline at end of file diff --git a/deploy/docker/schemas.py b/deploy/docker/schemas.py index 96196633..fe2d2801 100644 --- a/deploy/docker/schemas.py +++ b/deploy/docker/schemas.py @@ -9,6 +9,50 @@ class CrawlRequest(BaseModel): browser_config: Optional[Dict] = Field(default_factory=dict) crawler_config: Optional[Dict] = Field(default_factory=dict) + +class HookConfig(BaseModel): + """Configuration for user-provided hooks""" + code: Dict[str, str] = Field( + default_factory=dict, + description="Map of hook points to Python code strings" + ) + timeout: int = Field( + default=30, + ge=1, + le=120, + description="Timeout in seconds for each hook execution" + ) + + class Config: + schema_extra = { + "example": { + "code": { + "on_page_context_created": """ +async def hook(page, context, **kwargs): + # Block images to speed up crawling + await context.route("**/*.{png,jpg,jpeg,gif}", lambda route: route.abort()) + return page +""", + "before_retrieve_html": """ +async def hook(page, context, **kwargs): + # Scroll to load lazy content + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await page.wait_for_timeout(2000) + return page +""" + }, + "timeout": 30 + } + } + + +class CrawlRequestWithHooks(CrawlRequest): + """Extended crawl request with hooks support""" + hooks: Optional[HookConfig] = Field( + default=None, + description="Optional user-provided hook functions" + ) + class MarkdownRequest(BaseModel): """Request body for the /md endpoint.""" url: str = Field(..., description="Absolute http/https URL to fetch") diff --git a/deploy/docker/server.py b/deploy/docker/server.py index 12ebbb53..3dd8e58f 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -23,7 +23,7 @@ from api import ( stream_results ) from schemas import ( - CrawlRequest, + CrawlRequestWithHooks, MarkdownRequest, RawCode, HTMLRequest, @@ -414,6 +414,72 @@ async def get_schema(): "crawler": CrawlerRunConfig().dump()} +@app.get("/hooks/info") +async def get_hooks_info(): + """Get information about available hook points and their signatures""" + from hook_manager import UserHookManager + + hook_info = {} + for hook_point, params in UserHookManager.HOOK_SIGNATURES.items(): + hook_info[hook_point] = { + "parameters": params, + "description": get_hook_description(hook_point), + "example": get_hook_example(hook_point) + } + + return JSONResponse({ + "available_hooks": hook_info, + "timeout_limits": { + "min": 1, + "max": 120, + "default": 30 + } + }) + + +def get_hook_description(hook_point: str) -> str: + """Get description for each hook point""" + descriptions = { + "on_browser_created": "Called after browser instance is created", + "on_page_context_created": "Called after page and context are created - ideal for authentication", + "before_goto": "Called before navigating to the target URL", + "after_goto": "Called after navigation is complete", + "on_user_agent_updated": "Called when user agent is updated", + "on_execution_started": "Called when custom JavaScript execution begins", + "before_retrieve_html": "Called before retrieving the final HTML - ideal for scrolling", + "before_return_html": "Called just before returning the HTML content" + } + return descriptions.get(hook_point, "") + + +def get_hook_example(hook_point: str) -> str: + """Get example code for each hook point""" + examples = { + "on_page_context_created": """async def hook(page, context, **kwargs): + # Add authentication cookie + await context.add_cookies([{ + 'name': 'session', + 'value': 'my-session-id', + 'domain': '.example.com' + }]) + return page""", + + "before_retrieve_html": """async def hook(page, context, **kwargs): + # Scroll to load lazy content + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await page.wait_for_timeout(2000) + return page""", + + "before_goto": """async def hook(page, context, url, **kwargs): + # Set custom headers + await page.set_extra_http_headers({ + 'X-Custom-Header': 'value' + }) + return page""" + } + return examples.get(hook_point, "# Implement your hook logic here\nreturn page") + + @app.get(config["observability"]["health_check"]["endpoint"]) async def health(): return {"status": "ok", "timestamp": time.time(), "version": __version__} @@ -429,19 +495,30 @@ async def metrics(): @mcp_tool("crawl") async def crawl( request: Request, - crawl_request: CrawlRequest, + crawl_request: CrawlRequestWithHooks, _td: Dict = Depends(token_dep), ): """ Crawl a list of URLs and return the results as JSON. + Supports optional user-provided hook functions for customization. """ if not crawl_request.urls: raise HTTPException(400, "At least one URL required") + + # Prepare hooks config if provided + hooks_config = None + if crawl_request.hooks: + hooks_config = { + 'code': crawl_request.hooks.code, + 'timeout': crawl_request.hooks.timeout + } + res = await handle_crawl_request( urls=crawl_request.urls, browser_config=crawl_request.browser_config, crawler_config=crawl_request.crawler_config, config=config, + hooks_config=hooks_config ) return JSONResponse(res) @@ -450,25 +527,42 @@ async def crawl( @limiter.limit(config["rate_limiting"]["default_limit"]) async def crawl_stream( request: Request, - crawl_request: CrawlRequest, + crawl_request: CrawlRequestWithHooks, _td: Dict = Depends(token_dep), ): if not crawl_request.urls: raise HTTPException(400, "At least one URL required") - crawler, gen = await handle_stream_crawl_request( + + # Prepare hooks config if provided + hooks_config = None + if crawl_request.hooks: + hooks_config = { + 'code': crawl_request.hooks.code, + 'timeout': crawl_request.hooks.timeout + } + + crawler, gen, hooks_info = await handle_stream_crawl_request( urls=crawl_request.urls, browser_config=crawl_request.browser_config, crawler_config=crawl_request.crawler_config, config=config, + hooks_config=hooks_config ) + + # Add hooks info to response headers if available + headers = { + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Stream-Status": "active", + } + if hooks_info: + import json + headers["X-Hooks-Status"] = json.dumps(hooks_info['status']['status']) + return StreamingResponse( stream_results(crawler, gen), media_type="application/x-ndjson", - headers={ - "Cache-Control": "no-cache", - "Connection": "keep-alive", - "X-Stream-Status": "active", - }, + headers=headers, ) diff --git a/docs/examples/docker_hooks_examples.py b/docs/examples/docker_hooks_examples.py new file mode 100644 index 00000000..a9c94d03 --- /dev/null +++ b/docs/examples/docker_hooks_examples.py @@ -0,0 +1,513 @@ +#!/usr/bin/env python3 +""" +Comprehensive test demonstrating all hook types from hooks_example.py +adapted for the Docker API with real URLs +""" + +import requests +import json +import time +from typing import Dict, Any + +# API_BASE_URL = "http://localhost:11234" +API_BASE_URL = "http://localhost:11235" + + +def test_all_hooks_demo(): + """Demonstrate all 8 hook types with practical examples""" + print("=" * 70) + print("Testing: All Hooks Comprehensive Demo") + print("=" * 70) + + hooks_code = { + "on_browser_created": """ +async def hook(browser, **kwargs): + # Hook called after browser is created + print("[HOOK] on_browser_created - Browser is ready!") + # Browser-level configurations would go here + return browser +""", + + "on_page_context_created": """ +async def hook(page, context, **kwargs): + # Hook called after a new page and context are created + print("[HOOK] on_page_context_created - New page created!") + + # Set viewport size for consistent rendering + await page.set_viewport_size({"width": 1920, "height": 1080}) + + # Add cookies for the session (using httpbin.org domain) + await context.add_cookies([ + { + "name": "test_session", + "value": "abc123xyz", + "domain": ".httpbin.org", + "path": "/", + "httpOnly": True, + "secure": True + } + ]) + + # Block ads and tracking scripts to speed up crawling + await context.route("**/*.{png,jpg,jpeg,gif,webp,svg}", lambda route: route.abort()) + await context.route("**/analytics/*", lambda route: route.abort()) + await context.route("**/ads/*", lambda route: route.abort()) + + print("[HOOK] Viewport set, cookies added, and ads blocked") + return page +""", + + "on_user_agent_updated": """ +async def hook(page, context, user_agent, **kwargs): + # Hook called when user agent is updated + print(f"[HOOK] on_user_agent_updated - User agent: {user_agent[:50]}...") + return page +""", + + "before_goto": """ +async def hook(page, context, url, **kwargs): + # Hook called before navigating to each URL + print(f"[HOOK] before_goto - About to visit: {url}") + + # Add custom headers for the request + await page.set_extra_http_headers({ + "X-Custom-Header": "crawl4ai-test", + "Accept-Language": "en-US,en;q=0.9", + "DNT": "1" + }) + + return page +""", + + "after_goto": """ +async def hook(page, context, url, response, **kwargs): + # Hook called after navigating to each URL + print(f"[HOOK] after_goto - Successfully loaded: {url}") + + # Wait a moment for dynamic content to load + await page.wait_for_timeout(1000) + + # Check if specific elements exist (with error handling) + try: + # For httpbin.org, wait for body element + await page.wait_for_selector("body", timeout=2000) + print("[HOOK] Body element found and loaded") + except: + print("[HOOK] Timeout waiting for body, continuing anyway") + + return page +""", + + "on_execution_started": """ +async def hook(page, context, **kwargs): + # Hook called after custom JavaScript execution + print("[HOOK] on_execution_started - Custom JS executed!") + + # You could inject additional JavaScript here if needed + await page.evaluate("console.log('[INJECTED] Hook JS running');") + + return page +""", + + "before_retrieve_html": """ +async def hook(page, context, **kwargs): + # Hook called before retrieving the HTML content + print("[HOOK] before_retrieve_html - Preparing to get HTML") + + # Scroll to bottom to trigger lazy loading + await page.evaluate("window.scrollTo(0, document.body.scrollHeight);") + await page.wait_for_timeout(500) + + # Scroll back to top + await page.evaluate("window.scrollTo(0, 0);") + await page.wait_for_timeout(500) + + # One more scroll to middle for good measure + await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 2);") + + print("[HOOK] Scrolling completed for lazy-loaded content") + return page +""", + + "before_return_html": """ +async def hook(page, context, html, **kwargs): + # Hook called before returning the HTML content + print(f"[HOOK] before_return_html - HTML length: {len(html)} characters") + + # Log some page metrics + metrics = await page.evaluate('''() => { + return { + images: document.images.length, + links: document.links.length, + scripts: document.scripts.length + } + }''') + + print(f"[HOOK] Page metrics - Images: {metrics['images']}, Links: {metrics['links']}, Scripts: {metrics['scripts']}") + + return page +""" + } + + # Create request payload + payload = { + "urls": ["https://httpbin.org/html"], + "hooks": { + "code": hooks_code, + "timeout": 30 + }, + "crawler_config": { + "js_code": "window.scrollTo(0, document.body.scrollHeight);", + "wait_for": "body", + "cache_mode": "bypass" + } + } + + print("\nSending request with all 8 hooks...") + start_time = time.time() + + response = requests.post(f"{API_BASE_URL}/crawl", json=payload) + + elapsed_time = time.time() - start_time + print(f"Request completed in {elapsed_time:.2f} seconds") + + if response.status_code == 200: + data = response.json() + print("\n✅ Request successful!") + + # Check hooks execution + if 'hooks' in data: + hooks_info = data['hooks'] + print("\n📊 Hooks Execution Summary:") + print(f" Status: {hooks_info['status']['status']}") + print(f" Attached hooks: {len(hooks_info['status']['attached_hooks'])}") + + for hook_name in hooks_info['status']['attached_hooks']: + print(f" ✓ {hook_name}") + + if 'summary' in hooks_info: + summary = hooks_info['summary'] + print(f"\n📈 Execution Statistics:") + print(f" Total executions: {summary['total_executions']}") + print(f" Successful: {summary['successful']}") + print(f" Failed: {summary['failed']}") + print(f" Timed out: {summary['timed_out']}") + print(f" Success rate: {summary['success_rate']:.1f}%") + + if hooks_info.get('execution_log'): + print(f"\n📝 Execution Log:") + for log_entry in hooks_info['execution_log']: + status_icon = "✅" if log_entry['status'] == 'success' else "❌" + exec_time = log_entry.get('execution_time', 0) + print(f" {status_icon} {log_entry['hook_point']}: {exec_time:.3f}s") + + # Check crawl results + if 'results' in data and len(data['results']) > 0: + print(f"\n📄 Crawl Results:") + for result in data['results']: + print(f" URL: {result['url']}") + print(f" Success: {result.get('success', False)}") + if result.get('html'): + print(f" HTML length: {len(result['html'])} characters") + + else: + print(f"❌ Error: {response.status_code}") + try: + error_data = response.json() + print(f"Error details: {json.dumps(error_data, indent=2)}") + except: + print(f"Error text: {response.text[:500]}") + + +def test_authentication_flow(): + """Test a complete authentication flow with multiple hooks""" + print("\n" + "=" * 70) + print("Testing: Authentication Flow with Multiple Hooks") + print("=" * 70) + + hooks_code = { + "on_page_context_created": """ +async def hook(page, context, **kwargs): + print("[HOOK] Setting up authentication context") + + # Add authentication cookies + await context.add_cookies([ + { + "name": "auth_token", + "value": "fake_jwt_token_here", + "domain": ".httpbin.org", + "path": "/", + "httpOnly": True, + "secure": True + } + ]) + + # Set localStorage items (for SPA authentication) + await page.evaluate(''' + localStorage.setItem('user_id', '12345'); + localStorage.setItem('auth_time', new Date().toISOString()); + ''') + + return page +""", + + "before_goto": """ +async def hook(page, context, url, **kwargs): + print(f"[HOOK] Adding auth headers for {url}") + + # Add Authorization header + import base64 + credentials = base64.b64encode(b"user:passwd").decode('ascii') + + await page.set_extra_http_headers({ + 'Authorization': f'Basic {credentials}', + 'X-API-Key': 'test-api-key-123' + }) + + return page +""" + } + + payload = { + "urls": [ + "https://httpbin.org/basic-auth/user/passwd" + ], + "hooks": { + "code": hooks_code, + "timeout": 15 + } + } + + print("\nTesting authentication with httpbin endpoints...") + response = requests.post(f"{API_BASE_URL}/crawl", json=payload) + + if response.status_code == 200: + data = response.json() + print("✅ Authentication test completed") + + if 'results' in data: + for i, result in enumerate(data['results']): + print(f"\n URL {i+1}: {result['url']}") + if result.get('success'): + # Check for authentication success indicators + html_content = result.get('html', '') + if '"authenticated"' in html_content and 'true' in html_content: + print(" ✅ Authentication successful! Basic auth worked.") + else: + print(" ⚠️ Page loaded but auth status unclear") + else: + print(f" ❌ Failed: {result.get('error_message', 'Unknown error')}") + else: + print(f"❌ Error: {response.status_code}") + + +def test_performance_optimization_hooks(): + """Test hooks for performance optimization""" + print("\n" + "=" * 70) + print("Testing: Performance Optimization Hooks") + print("=" * 70) + + hooks_code = { + "on_page_context_created": """ +async def hook(page, context, **kwargs): + print("[HOOK] Optimizing page for performance") + + # Block resource-heavy content + await context.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort()) + await context.route("**/*.{woff,woff2,ttf,otf}", lambda route: route.abort()) + await context.route("**/*.{mp4,webm,ogg,mp3,wav}", lambda route: route.abort()) + await context.route("**/googletagmanager.com/*", lambda route: route.abort()) + await context.route("**/google-analytics.com/*", lambda route: route.abort()) + await context.route("**/doubleclick.net/*", lambda route: route.abort()) + await context.route("**/facebook.com/*", lambda route: route.abort()) + + # Disable animations and transitions + await page.add_style_tag(content=''' + *, *::before, *::after { + animation-duration: 0s !important; + animation-delay: 0s !important; + transition-duration: 0s !important; + transition-delay: 0s !important; + } + ''') + + print("[HOOK] Performance optimizations applied") + return page +""", + + "before_retrieve_html": """ +async def hook(page, context, **kwargs): + print("[HOOK] Removing unnecessary elements before extraction") + + # Remove ads, popups, and other unnecessary elements + await page.evaluate('''() => { + // Remove common ad containers + const adSelectors = [ + '.ad', '.ads', '.advertisement', '[id*="ad-"]', '[class*="ad-"]', + '.popup', '.modal', '.overlay', '.cookie-banner', '.newsletter-signup' + ]; + + adSelectors.forEach(selector => { + document.querySelectorAll(selector).forEach(el => el.remove()); + }); + + // Remove script tags to clean up HTML + document.querySelectorAll('script').forEach(el => el.remove()); + + // Remove style tags we don't need + document.querySelectorAll('style').forEach(el => el.remove()); + }''') + + return page +""" + } + + payload = { + "urls": ["https://httpbin.org/html"], + "hooks": { + "code": hooks_code, + "timeout": 10 + } + } + + print("\nTesting performance optimization hooks...") + start_time = time.time() + + response = requests.post(f"{API_BASE_URL}/crawl", json=payload) + + elapsed_time = time.time() - start_time + print(f"Request completed in {elapsed_time:.2f} seconds") + + if response.status_code == 200: + data = response.json() + print("✅ Performance optimization test completed") + + if 'results' in data and len(data['results']) > 0: + result = data['results'][0] + if result.get('html'): + print(f" HTML size: {len(result['html'])} characters") + print(" Resources blocked, ads removed, animations disabled") + else: + print(f"❌ Error: {response.status_code}") + + +def test_content_extraction_hooks(): + """Test hooks for intelligent content extraction""" + print("\n" + "=" * 70) + print("Testing: Content Extraction Hooks") + print("=" * 70) + + hooks_code = { + "after_goto": """ +async def hook(page, context, url, response, **kwargs): + print(f"[HOOK] Waiting for dynamic content on {url}") + + # Wait for any lazy-loaded content + await page.wait_for_timeout(2000) + + # Trigger any "Load More" buttons + try: + load_more = await page.query_selector('[class*="load-more"], [class*="show-more"], button:has-text("Load More")') + if load_more: + await load_more.click() + await page.wait_for_timeout(1000) + print("[HOOK] Clicked 'Load More' button") + except: + pass + + return page +""", + + "before_retrieve_html": """ +async def hook(page, context, **kwargs): + print("[HOOK] Extracting structured data") + + # Extract metadata + metadata = await page.evaluate('''() => { + const getMeta = (name) => { + const element = document.querySelector(`meta[name="${name}"], meta[property="${name}"]`); + return element ? element.getAttribute('content') : null; + }; + + return { + title: document.title, + description: getMeta('description') || getMeta('og:description'), + author: getMeta('author'), + keywords: getMeta('keywords'), + ogTitle: getMeta('og:title'), + ogImage: getMeta('og:image'), + canonical: document.querySelector('link[rel="canonical"]')?.href, + jsonLd: Array.from(document.querySelectorAll('script[type="application/ld+json"]')) + .map(el => el.textContent).filter(Boolean) + }; + }''') + + print(f"[HOOK] Extracted metadata: {json.dumps(metadata, indent=2)}") + + # Infinite scroll handling + for i in range(3): + await page.evaluate("window.scrollTo(0, document.body.scrollHeight);") + await page.wait_for_timeout(1000) + print(f"[HOOK] Scroll iteration {i+1}/3") + + return page +""" + } + + payload = { + "urls": ["https://httpbin.org/html", "https://httpbin.org/json"], + "hooks": { + "code": hooks_code, + "timeout": 20 + } + } + + print("\nTesting content extraction hooks...") + response = requests.post(f"{API_BASE_URL}/crawl", json=payload) + + if response.status_code == 200: + data = response.json() + print("✅ Content extraction test completed") + + if 'hooks' in data and 'summary' in data['hooks']: + summary = data['hooks']['summary'] + print(f" Hooks executed: {summary['successful']}/{summary['total_executions']}") + + if 'results' in data: + for result in data['results']: + print(f"\n URL: {result['url']}") + print(f" Success: {result.get('success', False)}") + else: + print(f"❌ Error: {response.status_code}") + + +def main(): + """Run comprehensive hook tests""" + print("🔧 Crawl4AI Docker API - Comprehensive Hooks Testing") + print("Based on docs/examples/hooks_example.py") + print("=" * 70) + + tests = [ + ("All Hooks Demo", test_all_hooks_demo), + ("Authentication Flow", test_authentication_flow), + ("Performance Optimization", test_performance_optimization_hooks), + ("Content Extraction", test_content_extraction_hooks), + ] + + for i, (name, test_func) in enumerate(tests, 1): + print(f"\n📌 Test {i}/{len(tests)}: {name}") + try: + test_func() + print(f"✅ {name} completed") + except Exception as e: + print(f"❌ {name} failed: {e}") + import traceback + traceback.print_exc() + + print("\n" + "=" * 70) + print("🎉 All comprehensive hook tests completed!") + print("=" * 70) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/docs/md_v2/core/docker-deployment.md b/docs/md_v2/core/docker-deployment.md index 544db1e2..a93b4409 100644 --- a/docs/md_v2/core/docker-deployment.md +++ b/docs/md_v2/core/docker-deployment.md @@ -405,6 +405,409 @@ Executes JavaScript snippets on the specified URL and returns the full crawl res --- +## User-Provided Hooks API + +The Docker API supports user-provided hook functions, allowing you to customize the crawling behavior by injecting your own Python code at specific points in the crawling pipeline. This powerful feature enables authentication, performance optimization, and custom content extraction without modifying the server code. + +> ⚠️ **IMPORTANT SECURITY WARNING**: +> - **Never use hooks with untrusted code or on untrusted websites** +> - **Be extremely careful when crawling sites that might be phishing or malicious** +> - **Hook code has access to page context and can interact with the website** +> - **Always validate and sanitize any data extracted through hooks** +> - **Never expose credentials or sensitive data in hook code** +> - **Consider running the Docker container in an isolated network when testing** + +### Hook Information Endpoint + +``` +GET /hooks/info +``` + +Returns information about available hook points and their signatures: + +```bash +curl http://localhost:11235/hooks/info +``` + +### Available Hook Points + +The API supports 8 hook points that match the local SDK: + +| Hook Point | Parameters | Description | Best Use Cases | +|------------|------------|-------------|----------------| +| `on_browser_created` | `browser` | After browser instance creation | Light setup tasks | +| `on_page_context_created` | `page, context` | After page/context creation | **Authentication, cookies, route blocking** | +| `before_goto` | `page, context, url` | Before navigating to URL | Custom headers, logging | +| `after_goto` | `page, context, url, response` | After navigation completes | Verification, waiting for elements | +| `on_user_agent_updated` | `page, context, user_agent` | When user agent changes | UA-specific logic | +| `on_execution_started` | `page, context` | When JS execution begins | JS-related setup | +| `before_retrieve_html` | `page, context` | Before getting final HTML | **Scrolling, lazy loading** | +| `before_return_html` | `page, context, html` | Before returning HTML | Final modifications, metrics | + +### Using Hooks in Requests + +Add hooks to any crawl request by including the `hooks` parameter: + +```json +{ + "urls": ["https://httpbin.org/html"], + "hooks": { + "code": { + "hook_point_name": "async def hook(...): ...", + "another_hook": "async def hook(...): ..." + }, + "timeout": 30 // Optional, default 30 seconds (max 120) + } +} +``` + +### Hook Examples with Real URLs + +#### 1. Authentication with Cookies (GitHub) + +```python +import requests + +# Example: Setting GitHub session cookie (use your actual session) +hooks_code = { + "on_page_context_created": """ +async def hook(page, context, **kwargs): + # Add authentication cookies for GitHub + # WARNING: Never hardcode real credentials! + await context.add_cookies([ + { + 'name': 'user_session', + 'value': 'your_github_session_token', # Replace with actual token + 'domain': '.github.com', + 'path': '/', + 'httpOnly': True, + 'secure': True, + 'sameSite': 'Lax' + } + ]) + return page +""" +} + +response = requests.post("http://localhost:11235/crawl", json={ + "urls": ["https://github.com/settings/profile"], # Protected page + "hooks": {"code": hooks_code, "timeout": 30} +}) +``` + +#### 2. Basic Authentication (httpbin.org for testing) + +```python +# Safe testing with httpbin.org (a service designed for HTTP testing) +hooks_code = { + "before_goto": """ +async def hook(page, context, url, **kwargs): + import base64 + # httpbin.org/basic-auth expects username="user" and password="passwd" + credentials = base64.b64encode(b"user:passwd").decode('ascii') + + await page.set_extra_http_headers({ + 'Authorization': f'Basic {credentials}' + }) + return page +""" +} + +response = requests.post("http://localhost:11235/crawl", json={ + "urls": ["https://httpbin.org/basic-auth/user/passwd"], + "hooks": {"code": hooks_code, "timeout": 15} +}) +``` + +#### 3. Performance Optimization (News Sites) + +```python +# Example: Optimizing crawling of news sites like CNN or BBC +hooks_code = { + "on_page_context_created": """ +async def hook(page, context, **kwargs): + # Block images, fonts, and media to speed up crawling + await context.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort()) + await context.route("**/*.{woff,woff2,ttf,otf,eot}", lambda route: route.abort()) + await context.route("**/*.{mp4,webm,ogg,mp3,wav,flac}", lambda route: route.abort()) + + # Block common tracking and ad domains + await context.route("**/googletagmanager.com/*", lambda route: route.abort()) + await context.route("**/google-analytics.com/*", lambda route: route.abort()) + await context.route("**/doubleclick.net/*", lambda route: route.abort()) + await context.route("**/facebook.com/tr/*", lambda route: route.abort()) + await context.route("**/amazon-adsystem.com/*", lambda route: route.abort()) + + # Disable CSS animations for faster rendering + await page.add_style_tag(content=''' + *, *::before, *::after { + animation-duration: 0s !important; + transition-duration: 0s !important; + } + ''') + + return page +""" +} + +response = requests.post("http://localhost:11235/crawl", json={ + "urls": ["https://www.bbc.com/news"], # Heavy news site + "hooks": {"code": hooks_code, "timeout": 30} +}) +``` + +#### 4. Handling Infinite Scroll (Twitter/X) + +```python +# Example: Scrolling on Twitter/X (requires authentication) +hooks_code = { + "before_retrieve_html": """ +async def hook(page, context, **kwargs): + # Scroll to load more tweets + previous_height = 0 + for i in range(5): # Limit scrolls to avoid infinite loop + current_height = await page.evaluate("document.body.scrollHeight") + if current_height == previous_height: + break # No more content to load + + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await page.wait_for_timeout(2000) # Wait for content to load + previous_height = current_height + + return page +""" +} + +# Note: Twitter requires authentication for most content +response = requests.post("http://localhost:11235/crawl", json={ + "urls": ["https://twitter.com/nasa"], # Public profile + "hooks": {"code": hooks_code, "timeout": 30} +}) +``` + +#### 5. E-commerce Login (Example Pattern) + +```python +# SECURITY WARNING: This is a pattern example. +# Never use real credentials in code! +# Always use environment variables or secure vaults. + +hooks_code = { + "on_page_context_created": """ +async def hook(page, context, **kwargs): + # Example pattern for e-commerce sites + # DO NOT use real credentials here! + + # Navigate to login page first + await page.goto("https://example-shop.com/login") + + # Wait for login form to load + await page.wait_for_selector("#email", timeout=5000) + + # Fill login form (use environment variables in production!) + await page.fill("#email", "test@example.com") # Never use real email + await page.fill("#password", "test_password") # Never use real password + + # Handle "Remember Me" checkbox if present + try: + await page.uncheck("#remember_me") # Don't remember on shared systems + except: + pass + + # Submit form + await page.click("button[type='submit']") + + # Wait for redirect after login + await page.wait_for_url("**/account/**", timeout=10000) + + return page +""" +} +``` + +#### 6. Extracting Structured Data (Wikipedia) + +```python +# Safe example using Wikipedia +hooks_code = { + "after_goto": """ +async def hook(page, context, url, response, **kwargs): + # Wait for Wikipedia content to load + await page.wait_for_selector("#content", timeout=5000) + return page +""", + + "before_retrieve_html": """ +async def hook(page, context, **kwargs): + # Extract structured data from Wikipedia infobox + metadata = await page.evaluate('''() => { + const infobox = document.querySelector('.infobox'); + if (!infobox) return null; + + const data = {}; + const rows = infobox.querySelectorAll('tr'); + + rows.forEach(row => { + const header = row.querySelector('th'); + const value = row.querySelector('td'); + if (header && value) { + data[header.innerText.trim()] = value.innerText.trim(); + } + }); + + return data; + }''') + + if metadata: + console.log("Extracted metadata:", metadata) + + return page +""" +} + +response = requests.post("http://localhost:11235/crawl", json={ + "urls": ["https://en.wikipedia.org/wiki/Python_(programming_language)"], + "hooks": {"code": hooks_code, "timeout": 20} +}) +``` + +### Security Best Practices + +> 🔒 **Critical Security Guidelines**: + +1. **Never Trust User Input**: If accepting hook code from users, always validate and sandbox it +2. **Avoid Phishing Sites**: Never use hooks on suspicious or unverified websites +3. **Protect Credentials**: + - Never hardcode passwords, tokens, or API keys in hook code + - Use environment variables or secure secret management + - Rotate credentials regularly +4. **Network Isolation**: Run the Docker container in an isolated network when testing +5. **Audit Hook Code**: Always review hook code before execution +6. **Limit Permissions**: Use the least privileged access needed +7. **Monitor Execution**: Check hook execution logs for suspicious behavior +8. **Timeout Protection**: Always set reasonable timeouts (default 30s) + +### Hook Response Information + +When hooks are used, the response includes detailed execution information: + +```json +{ + "success": true, + "results": [...], + "hooks": { + "status": { + "status": "success", // or "partial" or "failed" + "attached_hooks": ["on_page_context_created", "before_retrieve_html"], + "validation_errors": [], + "successfully_attached": 2, + "failed_validation": 0 + }, + "execution_log": [ + { + "hook_point": "on_page_context_created", + "status": "success", + "execution_time": 0.523, + "timestamp": 1234567890.123 + } + ], + "errors": [], // Any runtime errors + "summary": { + "total_executions": 2, + "successful": 2, + "failed": 0, + "timed_out": 0, + "success_rate": 100.0 + } + } +} +``` + +### Error Handling + +The hooks system is designed to be resilient: + +1. **Validation Errors**: Caught before execution (syntax errors, wrong parameters) +2. **Runtime Errors**: Handled gracefully - crawl continues with original page object +3. **Timeout Protection**: Hooks automatically terminated after timeout (configurable 1-120s) + +### Complete Example: Safe Multi-Hook Crawling + +```python +import requests +import json +import os + +# Safe example using httpbin.org for testing +hooks_code = { + "on_page_context_created": """ +async def hook(page, context, **kwargs): + # Set viewport and test cookies + await page.set_viewport_size({"width": 1920, "height": 1080}) + await context.add_cookies([ + {"name": "test_cookie", "value": "test_value", "domain": ".httpbin.org", "path": "/"} + ]) + + # Block unnecessary resources for httpbin + await context.route("**/*.{png,jpg,jpeg}", lambda route: route.abort()) + return page +""", + + "before_goto": """ +async def hook(page, context, url, **kwargs): + # Add custom headers for testing + await page.set_extra_http_headers({ + "X-Test-Header": "crawl4ai-test", + "Accept-Language": "en-US,en;q=0.9" + }) + print(f"[HOOK] Navigating to: {url}") + return page +""", + + "before_retrieve_html": """ +async def hook(page, context, **kwargs): + # Simple scroll for any lazy-loaded content + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await page.wait_for_timeout(1000) + return page +""" +} + +# Make the request to safe testing endpoints +response = requests.post("http://localhost:11235/crawl", json={ + "urls": [ + "https://httpbin.org/html", + "https://httpbin.org/json" + ], + "hooks": { + "code": hooks_code, + "timeout": 30 + }, + "crawler_config": { + "cache_mode": "bypass" + } +}) + +# Check results +if response.status_code == 200: + data = response.json() + + # Check hook execution + if data['hooks']['status']['status'] == 'success': + print(f"✅ All {len(data['hooks']['status']['attached_hooks'])} hooks executed successfully") + print(f"Execution stats: {data['hooks']['summary']}") + + # Process crawl results + for result in data['results']: + print(f"Crawled: {result['url']} - Success: {result['success']}") +else: + print(f"Error: {response.status_code}") +``` + +> 💡 **Remember**: Always test your hooks on safe, known websites first before using them on production sites. Never crawl sites that you don't have permission to access or that might be malicious. + +--- + ## Dockerfile Parameters You can customize the image build process using build arguments (`--build-arg`). These are typically used via `docker buildx build` or within the `docker-compose.yml` file. diff --git a/tests/docker/test_hooks_client.py b/tests/docker/test_hooks_client.py new file mode 100644 index 00000000..bfac353f --- /dev/null +++ b/tests/docker/test_hooks_client.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +""" +Test client for demonstrating user-provided hooks in Crawl4AI Docker API +""" + +import requests +import json +from typing import Dict, Any + + +API_BASE_URL = "http://localhost:11234" # Adjust if needed + + +def test_hooks_info(): + """Get information about available hooks""" + print("=" * 70) + print("Testing: GET /hooks/info") + print("=" * 70) + + response = requests.get(f"{API_BASE_URL}/hooks/info") + if response.status_code == 200: + data = response.json() + print("Available Hook Points:") + for hook, info in data['available_hooks'].items(): + print(f"\n{hook}:") + print(f" Parameters: {', '.join(info['parameters'])}") + print(f" Description: {info['description']}") + else: + print(f"Error: {response.status_code}") + print(response.text) + + +def test_basic_crawl_with_hooks(): + """Test basic crawling with user-provided hooks""" + print("\n" + "=" * 70) + print("Testing: POST /crawl with hooks") + print("=" * 70) + + # Define hooks as Python code strings + hooks_code = { + "on_page_context_created": """ +async def hook(page, context, **kwargs): + print("Hook: Setting up page context") + # Block images to speed up crawling + await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort()) + print("Hook: Images blocked") + return page +""", + + "before_retrieve_html": """ +async def hook(page, context, **kwargs): + print("Hook: Before retrieving HTML") + # Scroll to bottom to load lazy content + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await page.wait_for_timeout(1000) + print("Hook: Scrolled to bottom") + return page +""", + + "before_goto": """ +async def hook(page, context, url, **kwargs): + print(f"Hook: About to navigate to {url}") + # Add custom headers + await page.set_extra_http_headers({ + 'X-Test-Header': 'crawl4ai-hooks-test' + }) + return page +""" + } + + # Create request payload + payload = { + "urls": ["https://httpbin.org/html"], + "hooks": { + "code": hooks_code, + "timeout": 30 + } + } + + print("Sending request with hooks...") + response = requests.post(f"{API_BASE_URL}/crawl", json=payload) + + if response.status_code == 200: + data = response.json() + print("\n✅ Crawl successful!") + + # Check hooks status + if 'hooks' in data: + hooks_info = data['hooks'] + print("\nHooks Execution Summary:") + print(f" Status: {hooks_info['status']['status']}") + print(f" Attached hooks: {', '.join(hooks_info['status']['attached_hooks'])}") + + if hooks_info['status']['validation_errors']: + print("\n⚠️ Validation Errors:") + for error in hooks_info['status']['validation_errors']: + print(f" - {error['hook_point']}: {error['error']}") + + if 'summary' in hooks_info: + summary = hooks_info['summary'] + print(f"\nExecution Statistics:") + print(f" Total executions: {summary['total_executions']}") + print(f" Successful: {summary['successful']}") + print(f" Failed: {summary['failed']}") + print(f" Timed out: {summary['timed_out']}") + print(f" Success rate: {summary['success_rate']:.1f}%") + + if hooks_info['execution_log']: + print("\nExecution Log:") + for log_entry in hooks_info['execution_log']: + status_icon = "✅" if log_entry['status'] == 'success' else "❌" + print(f" {status_icon} {log_entry['hook_point']}: {log_entry['status']} ({log_entry.get('execution_time', 0):.2f}s)") + + if hooks_info['errors']: + print("\n❌ Hook Errors:") + for error in hooks_info['errors']: + print(f" - {error['hook_point']}: {error['error']}") + + # Show crawl results + if 'results' in data: + print(f"\nCrawled {len(data['results'])} URL(s)") + for result in data['results']: + print(f" - {result['url']}: {'✅' if result['success'] else '❌'}") + + else: + print(f"❌ Error: {response.status_code}") + print(response.text) + + +def test_invalid_hook(): + """Test with an invalid hook to see error handling""" + print("\n" + "=" * 70) + print("Testing: Invalid hook handling") + print("=" * 70) + + # Intentionally broken hook + hooks_code = { + "on_page_context_created": """ +def hook(page, context): # Missing async! + return page +""", + + "before_retrieve_html": """ +async def hook(page, context, **kwargs): + # This will cause an error + await page.non_existent_method() + return page +""" + } + + payload = { + "urls": ["https://httpbin.org/html"], + "hooks": { + "code": hooks_code, + "timeout": 5 + } + } + + print("Sending request with invalid hooks...") + response = requests.post(f"{API_BASE_URL}/crawl", json=payload) + + if response.status_code == 200: + data = response.json() + + if 'hooks' in data: + hooks_info = data['hooks'] + print(f"\nHooks Status: {hooks_info['status']['status']}") + + if hooks_info['status']['validation_errors']: + print("\n✅ Validation caught errors (as expected):") + for error in hooks_info['status']['validation_errors']: + print(f" - {error['hook_point']}: {error['error']}") + + if hooks_info['errors']: + print("\n✅ Runtime errors handled gracefully:") + for error in hooks_info['errors']: + print(f" - {error['hook_point']}: {error['error']}") + + # The crawl should still succeed despite hook errors + if data.get('success'): + print("\n✅ Crawl succeeded despite hook errors (error isolation working!)") + + else: + print(f"Error: {response.status_code}") + print(response.text) + + +def test_authentication_hook(): + """Test authentication using hooks""" + print("\n" + "=" * 70) + print("Testing: Authentication with hooks") + print("=" * 70) + + hooks_code = { + "before_goto": """ +async def hook(page, context, url, **kwargs): + # For httpbin.org basic auth test, set Authorization header + import base64 + + # httpbin.org/basic-auth/user/passwd expects username="user" and password="passwd" + credentials = base64.b64encode(b"user:passwd").decode('ascii') + + await page.set_extra_http_headers({ + 'Authorization': f'Basic {credentials}' + }) + + print(f"Hook: Set Authorization header for {url}") + return page +""", + "on_page_context_created": """ +async def hook(page, context, **kwargs): + # Example: Add cookies for session tracking + await context.add_cookies([ + { + 'name': 'session_id', + 'value': 'test_session_123', + 'domain': '.httpbin.org', + 'path': '/', + 'httpOnly': True, + 'secure': True + } + ]) + + print("Hook: Added session cookie") + return page +""" + } + + payload = { + "urls": ["https://httpbin.org/basic-auth/user/passwd"], + "hooks": { + "code": hooks_code, + "timeout": 30 + } + } + + print("Sending request with authentication hook...") + response = requests.post(f"{API_BASE_URL}/crawl", json=payload) + + if response.status_code == 200: + data = response.json() + if data.get('success'): + print("✅ Crawl with authentication hook successful") + + # Check if hooks executed + if 'hooks' in data: + hooks_info = data['hooks'] + if hooks_info.get('summary', {}).get('successful', 0) > 0: + print(f"✅ Authentication hooks executed: {hooks_info['summary']['successful']} successful") + + # Check for any hook errors + if hooks_info.get('errors'): + print("⚠️ Hook errors:") + for error in hooks_info['errors']: + print(f" - {error}") + + # Check if authentication worked by looking at the result + if 'results' in data and len(data['results']) > 0: + result = data['results'][0] + if result.get('success'): + print("✅ Page crawled successfully (authentication worked!)") + # httpbin.org/basic-auth returns JSON with authenticated=true when successful + if 'authenticated' in str(result.get('html', '')): + print("✅ Authentication confirmed in response content") + else: + print(f"❌ Crawl failed: {result.get('error_message', 'Unknown error')}") + else: + print("❌ Request failed") + print(f"Response: {json.dumps(data, indent=2)}") + else: + print(f"❌ Error: {response.status_code}") + try: + error_data = response.json() + print(f"Error details: {json.dumps(error_data, indent=2)}") + except: + print(f"Error text: {response.text[:500]}") + + +def test_streaming_with_hooks(): + """Test streaming endpoint with hooks""" + print("\n" + "=" * 70) + print("Testing: POST /crawl/stream with hooks") + print("=" * 70) + + hooks_code = { + "before_retrieve_html": """ +async def hook(page, context, **kwargs): + await page.evaluate("document.querySelectorAll('img').forEach(img => img.remove())") + return page +""" + } + + payload = { + "urls": ["https://httpbin.org/html", "https://httpbin.org/json"], + "hooks": { + "code": hooks_code, + "timeout": 10 + } + } + + print("Sending streaming request with hooks...") + + with requests.post(f"{API_BASE_URL}/crawl/stream", json=payload, stream=True) as response: + if response.status_code == 200: + # Check headers for hooks status + hooks_status = response.headers.get('X-Hooks-Status') + if hooks_status: + print(f"Hooks Status (from header): {hooks_status}") + + print("\nStreaming results:") + for line in response.iter_lines(): + if line: + try: + result = json.loads(line) + if 'url' in result: + print(f" Received: {result['url']}") + elif 'status' in result: + print(f" Stream status: {result['status']}") + except json.JSONDecodeError: + print(f" Raw: {line.decode()}") + else: + print(f"Error: {response.status_code}") + + +def test_basic_without_hooks(): + """Test basic crawl without hooks""" + print("\n" + "=" * 70) + print("Testing: POST /crawl with no hooks") + print("=" * 70) + + payload = { + "urls": ["https://httpbin.org/html", "https://httpbin.org/json"] + } + + response = requests.post(f"{API_BASE_URL}/crawl", json=payload) + if response.status_code == 200: + data = response.json() + print(f"Response: {json.dumps(data, indent=2)}") + else: + print(f"Error: {response.status_code}") + + +def main(): + """Run all tests""" + print("🔧 Crawl4AI Docker API - Hooks Testing") + print("=" * 70) + + # Test 1: Get hooks information + # test_hooks_info() + + # Test 2: Basic crawl with hooks + # test_basic_crawl_with_hooks() + + # Test 3: Invalid hooks (error handling) + test_invalid_hook() + + # # Test 4: Authentication hook + # test_authentication_hook() + + # # Test 5: Streaming with hooks + # test_streaming_with_hooks() + + # # Test 6: Basic crawl without hooks + # test_basic_without_hooks() + + print("\n" + "=" * 70) + print("✅ All tests completed!") + print("=" * 70) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/docker/test_hooks_comprehensive.py b/tests/docker/test_hooks_comprehensive.py new file mode 100644 index 00000000..37783d60 --- /dev/null +++ b/tests/docker/test_hooks_comprehensive.py @@ -0,0 +1,512 @@ +#!/usr/bin/env python3 +""" +Comprehensive test demonstrating all hook types from hooks_example.py +adapted for the Docker API with real URLs +""" + +import requests +import json +import time +from typing import Dict, Any + +API_BASE_URL = "http://localhost:11234" + + +def test_all_hooks_demo(): + """Demonstrate all 8 hook types with practical examples""" + print("=" * 70) + print("Testing: All Hooks Comprehensive Demo") + print("=" * 70) + + hooks_code = { + "on_browser_created": """ +async def hook(browser, **kwargs): + # Hook called after browser is created + print("[HOOK] on_browser_created - Browser is ready!") + # Browser-level configurations would go here + return browser +""", + + "on_page_context_created": """ +async def hook(page, context, **kwargs): + # Hook called after a new page and context are created + print("[HOOK] on_page_context_created - New page created!") + + # Set viewport size for consistent rendering + await page.set_viewport_size({"width": 1920, "height": 1080}) + + # Add cookies for the session (using httpbin.org domain) + await context.add_cookies([ + { + "name": "test_session", + "value": "abc123xyz", + "domain": ".httpbin.org", + "path": "/", + "httpOnly": True, + "secure": True + } + ]) + + # Block ads and tracking scripts to speed up crawling + await context.route("**/*.{png,jpg,jpeg,gif,webp,svg}", lambda route: route.abort()) + await context.route("**/analytics/*", lambda route: route.abort()) + await context.route("**/ads/*", lambda route: route.abort()) + + print("[HOOK] Viewport set, cookies added, and ads blocked") + return page +""", + + "on_user_agent_updated": """ +async def hook(page, context, user_agent, **kwargs): + # Hook called when user agent is updated + print(f"[HOOK] on_user_agent_updated - User agent: {user_agent[:50]}...") + return page +""", + + "before_goto": """ +async def hook(page, context, url, **kwargs): + # Hook called before navigating to each URL + print(f"[HOOK] before_goto - About to visit: {url}") + + # Add custom headers for the request + await page.set_extra_http_headers({ + "X-Custom-Header": "crawl4ai-test", + "Accept-Language": "en-US,en;q=0.9", + "DNT": "1" + }) + + return page +""", + + "after_goto": """ +async def hook(page, context, url, response, **kwargs): + # Hook called after navigating to each URL + print(f"[HOOK] after_goto - Successfully loaded: {url}") + + # Wait a moment for dynamic content to load + await page.wait_for_timeout(1000) + + # Check if specific elements exist (with error handling) + try: + # For httpbin.org, wait for body element + await page.wait_for_selector("body", timeout=2000) + print("[HOOK] Body element found and loaded") + except: + print("[HOOK] Timeout waiting for body, continuing anyway") + + return page +""", + + "on_execution_started": """ +async def hook(page, context, **kwargs): + # Hook called after custom JavaScript execution + print("[HOOK] on_execution_started - Custom JS executed!") + + # You could inject additional JavaScript here if needed + await page.evaluate("console.log('[INJECTED] Hook JS running');") + + return page +""", + + "before_retrieve_html": """ +async def hook(page, context, **kwargs): + # Hook called before retrieving the HTML content + print("[HOOK] before_retrieve_html - Preparing to get HTML") + + # Scroll to bottom to trigger lazy loading + await page.evaluate("window.scrollTo(0, document.body.scrollHeight);") + await page.wait_for_timeout(500) + + # Scroll back to top + await page.evaluate("window.scrollTo(0, 0);") + await page.wait_for_timeout(500) + + # One more scroll to middle for good measure + await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 2);") + + print("[HOOK] Scrolling completed for lazy-loaded content") + return page +""", + + "before_return_html": """ +async def hook(page, context, html, **kwargs): + # Hook called before returning the HTML content + print(f"[HOOK] before_return_html - HTML length: {len(html)} characters") + + # Log some page metrics + metrics = await page.evaluate('''() => { + return { + images: document.images.length, + links: document.links.length, + scripts: document.scripts.length + } + }''') + + print(f"[HOOK] Page metrics - Images: {metrics['images']}, Links: {metrics['links']}, Scripts: {metrics['scripts']}") + + return page +""" + } + + # Create request payload + payload = { + "urls": ["https://httpbin.org/html"], + "hooks": { + "code": hooks_code, + "timeout": 30 + }, + "crawler_config": { + "js_code": "window.scrollTo(0, document.body.scrollHeight);", + "wait_for": "body", + "cache_mode": "bypass" + } + } + + print("\nSending request with all 8 hooks...") + start_time = time.time() + + response = requests.post(f"{API_BASE_URL}/crawl", json=payload) + + elapsed_time = time.time() - start_time + print(f"Request completed in {elapsed_time:.2f} seconds") + + if response.status_code == 200: + data = response.json() + print("\n✅ Request successful!") + + # Check hooks execution + if 'hooks' in data: + hooks_info = data['hooks'] + print("\n📊 Hooks Execution Summary:") + print(f" Status: {hooks_info['status']['status']}") + print(f" Attached hooks: {len(hooks_info['status']['attached_hooks'])}") + + for hook_name in hooks_info['status']['attached_hooks']: + print(f" ✓ {hook_name}") + + if 'summary' in hooks_info: + summary = hooks_info['summary'] + print(f"\n📈 Execution Statistics:") + print(f" Total executions: {summary['total_executions']}") + print(f" Successful: {summary['successful']}") + print(f" Failed: {summary['failed']}") + print(f" Timed out: {summary['timed_out']}") + print(f" Success rate: {summary['success_rate']:.1f}%") + + if hooks_info.get('execution_log'): + print(f"\n📝 Execution Log:") + for log_entry in hooks_info['execution_log']: + status_icon = "✅" if log_entry['status'] == 'success' else "❌" + exec_time = log_entry.get('execution_time', 0) + print(f" {status_icon} {log_entry['hook_point']}: {exec_time:.3f}s") + + # Check crawl results + if 'results' in data and len(data['results']) > 0: + print(f"\n📄 Crawl Results:") + for result in data['results']: + print(f" URL: {result['url']}") + print(f" Success: {result.get('success', False)}") + if result.get('html'): + print(f" HTML length: {len(result['html'])} characters") + + else: + print(f"❌ Error: {response.status_code}") + try: + error_data = response.json() + print(f"Error details: {json.dumps(error_data, indent=2)}") + except: + print(f"Error text: {response.text[:500]}") + + +def test_authentication_flow(): + """Test a complete authentication flow with multiple hooks""" + print("\n" + "=" * 70) + print("Testing: Authentication Flow with Multiple Hooks") + print("=" * 70) + + hooks_code = { + "on_page_context_created": """ +async def hook(page, context, **kwargs): + print("[HOOK] Setting up authentication context") + + # Add authentication cookies + await context.add_cookies([ + { + "name": "auth_token", + "value": "fake_jwt_token_here", + "domain": ".httpbin.org", + "path": "/", + "httpOnly": True, + "secure": True + } + ]) + + # Set localStorage items (for SPA authentication) + await page.evaluate(''' + localStorage.setItem('user_id', '12345'); + localStorage.setItem('auth_time', new Date().toISOString()); + ''') + + return page +""", + + "before_goto": """ +async def hook(page, context, url, **kwargs): + print(f"[HOOK] Adding auth headers for {url}") + + # Add Authorization header + import base64 + credentials = base64.b64encode(b"user:passwd").decode('ascii') + + await page.set_extra_http_headers({ + 'Authorization': f'Basic {credentials}', + 'X-API-Key': 'test-api-key-123' + }) + + return page +""" + } + + payload = { + "urls": [ + "https://httpbin.org/basic-auth/user/passwd" + ], + "hooks": { + "code": hooks_code, + "timeout": 15 + } + } + + print("\nTesting authentication with httpbin endpoints...") + response = requests.post(f"{API_BASE_URL}/crawl", json=payload) + + if response.status_code == 200: + data = response.json() + print("✅ Authentication test completed") + + if 'results' in data: + for i, result in enumerate(data['results']): + print(f"\n URL {i+1}: {result['url']}") + if result.get('success'): + # Check for authentication success indicators + html_content = result.get('html', '') + if '"authenticated"' in html_content and 'true' in html_content: + print(" ✅ Authentication successful! Basic auth worked.") + else: + print(" ⚠️ Page loaded but auth status unclear") + else: + print(f" ❌ Failed: {result.get('error_message', 'Unknown error')}") + else: + print(f"❌ Error: {response.status_code}") + + +def test_performance_optimization_hooks(): + """Test hooks for performance optimization""" + print("\n" + "=" * 70) + print("Testing: Performance Optimization Hooks") + print("=" * 70) + + hooks_code = { + "on_page_context_created": """ +async def hook(page, context, **kwargs): + print("[HOOK] Optimizing page for performance") + + # Block resource-heavy content + await context.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort()) + await context.route("**/*.{woff,woff2,ttf,otf}", lambda route: route.abort()) + await context.route("**/*.{mp4,webm,ogg,mp3,wav}", lambda route: route.abort()) + await context.route("**/googletagmanager.com/*", lambda route: route.abort()) + await context.route("**/google-analytics.com/*", lambda route: route.abort()) + await context.route("**/doubleclick.net/*", lambda route: route.abort()) + await context.route("**/facebook.com/*", lambda route: route.abort()) + + # Disable animations and transitions + await page.add_style_tag(content=''' + *, *::before, *::after { + animation-duration: 0s !important; + animation-delay: 0s !important; + transition-duration: 0s !important; + transition-delay: 0s !important; + } + ''') + + print("[HOOK] Performance optimizations applied") + return page +""", + + "before_retrieve_html": """ +async def hook(page, context, **kwargs): + print("[HOOK] Removing unnecessary elements before extraction") + + # Remove ads, popups, and other unnecessary elements + await page.evaluate('''() => { + // Remove common ad containers + const adSelectors = [ + '.ad', '.ads', '.advertisement', '[id*="ad-"]', '[class*="ad-"]', + '.popup', '.modal', '.overlay', '.cookie-banner', '.newsletter-signup' + ]; + + adSelectors.forEach(selector => { + document.querySelectorAll(selector).forEach(el => el.remove()); + }); + + // Remove script tags to clean up HTML + document.querySelectorAll('script').forEach(el => el.remove()); + + // Remove style tags we don't need + document.querySelectorAll('style').forEach(el => el.remove()); + }''') + + return page +""" + } + + payload = { + "urls": ["https://httpbin.org/html"], + "hooks": { + "code": hooks_code, + "timeout": 10 + } + } + + print("\nTesting performance optimization hooks...") + start_time = time.time() + + response = requests.post(f"{API_BASE_URL}/crawl", json=payload) + + elapsed_time = time.time() - start_time + print(f"Request completed in {elapsed_time:.2f} seconds") + + if response.status_code == 200: + data = response.json() + print("✅ Performance optimization test completed") + + if 'results' in data and len(data['results']) > 0: + result = data['results'][0] + if result.get('html'): + print(f" HTML size: {len(result['html'])} characters") + print(" Resources blocked, ads removed, animations disabled") + else: + print(f"❌ Error: {response.status_code}") + + +def test_content_extraction_hooks(): + """Test hooks for intelligent content extraction""" + print("\n" + "=" * 70) + print("Testing: Content Extraction Hooks") + print("=" * 70) + + hooks_code = { + "after_goto": """ +async def hook(page, context, url, response, **kwargs): + print(f"[HOOK] Waiting for dynamic content on {url}") + + # Wait for any lazy-loaded content + await page.wait_for_timeout(2000) + + # Trigger any "Load More" buttons + try: + load_more = await page.query_selector('[class*="load-more"], [class*="show-more"], button:has-text("Load More")') + if load_more: + await load_more.click() + await page.wait_for_timeout(1000) + print("[HOOK] Clicked 'Load More' button") + except: + pass + + return page +""", + + "before_retrieve_html": """ +async def hook(page, context, **kwargs): + print("[HOOK] Extracting structured data") + + # Extract metadata + metadata = await page.evaluate('''() => { + const getMeta = (name) => { + const element = document.querySelector(`meta[name="${name}"], meta[property="${name}"]`); + return element ? element.getAttribute('content') : null; + }; + + return { + title: document.title, + description: getMeta('description') || getMeta('og:description'), + author: getMeta('author'), + keywords: getMeta('keywords'), + ogTitle: getMeta('og:title'), + ogImage: getMeta('og:image'), + canonical: document.querySelector('link[rel="canonical"]')?.href, + jsonLd: Array.from(document.querySelectorAll('script[type="application/ld+json"]')) + .map(el => el.textContent).filter(Boolean) + }; + }''') + + print(f"[HOOK] Extracted metadata: {json.dumps(metadata, indent=2)}") + + # Infinite scroll handling + for i in range(3): + await page.evaluate("window.scrollTo(0, document.body.scrollHeight);") + await page.wait_for_timeout(1000) + print(f"[HOOK] Scroll iteration {i+1}/3") + + return page +""" + } + + payload = { + "urls": ["https://httpbin.org/html", "https://httpbin.org/json"], + "hooks": { + "code": hooks_code, + "timeout": 20 + } + } + + print("\nTesting content extraction hooks...") + response = requests.post(f"{API_BASE_URL}/crawl", json=payload) + + if response.status_code == 200: + data = response.json() + print("✅ Content extraction test completed") + + if 'hooks' in data and 'summary' in data['hooks']: + summary = data['hooks']['summary'] + print(f" Hooks executed: {summary['successful']}/{summary['total_executions']}") + + if 'results' in data: + for result in data['results']: + print(f"\n URL: {result['url']}") + print(f" Success: {result.get('success', False)}") + else: + print(f"❌ Error: {response.status_code}") + + +def main(): + """Run comprehensive hook tests""" + print("🔧 Crawl4AI Docker API - Comprehensive Hooks Testing") + print("Based on docs/examples/hooks_example.py") + print("=" * 70) + + tests = [ + ("All Hooks Demo", test_all_hooks_demo), + ("Authentication Flow", test_authentication_flow), + ("Performance Optimization", test_performance_optimization_hooks), + ("Content Extraction", test_content_extraction_hooks), + ] + + for i, (name, test_func) in enumerate(tests, 1): + print(f"\n📌 Test {i}/{len(tests)}: {name}") + try: + test_func() + print(f"✅ {name} completed") + except Exception as e: + print(f"❌ {name} failed: {e}") + import traceback + traceback.print_exc() + + print("\n" + "=" * 70) + print("🎉 All comprehensive hook tests completed!") + print("=" * 70) + + +if __name__ == "__main__": + main() \ No newline at end of file From 88a9fbbb7e507006bd5492c4bb83e0b4efa0614f Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 11 Aug 2025 18:16:57 +0800 Subject: [PATCH 004/119] fix(deep-crawl): BestFirst priority inversion; remove pre-scoring truncation. ref #1253 Use negative scores in PQ to visit high-score URLs first and drop link cap prior to scoring; add test for ordering. --- crawl4ai/deep_crawling/bff_strategy.py | 12 +-- tests/general/test_bff_scoring.py | 117 +++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 tests/general/test_bff_scoring.py diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.py index 7779c9f4..82f8e184 100644 --- a/crawl4ai/deep_crawling/bff_strategy.py +++ b/crawl4ai/deep_crawling/bff_strategy.py @@ -116,11 +116,6 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy): valid_links.append(base_url) - # If we have more valid links than capacity, limit them - if len(valid_links) > remaining_capacity: - valid_links = valid_links[:remaining_capacity] - self.logger.info(f"Limiting to {remaining_capacity} URLs due to max_pages limit") - # Record the new depths and add to next_links for url in valid_links: depths[url] = new_depth @@ -140,7 +135,8 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy): """ queue: asyncio.PriorityQueue = asyncio.PriorityQueue() # Push the initial URL with score 0 and depth 0. - await queue.put((0, 0, start_url, None)) + initial_score = self.url_scorer.score(start_url) if self.url_scorer else 0 + await queue.put((-initial_score, 0, start_url, None)) visited: Set[str] = set() depths: Dict[str, int] = {start_url: 0} @@ -187,7 +183,7 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy): result.metadata = result.metadata or {} result.metadata["depth"] = depth result.metadata["parent_url"] = parent_url - result.metadata["score"] = score + result.metadata["score"] = -score # Count only successful crawls toward max_pages limit if result.success: @@ -208,7 +204,7 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy): for new_url, new_parent in new_links: new_depth = depths.get(new_url, depth + 1) new_score = self.url_scorer.score(new_url) if self.url_scorer else 0 - await queue.put((new_score, new_depth, new_url, new_parent)) + await queue.put((-new_score, new_depth, new_url, new_parent)) # End of crawl. diff --git a/tests/general/test_bff_scoring.py b/tests/general/test_bff_scoring.py new file mode 100644 index 00000000..d663d944 --- /dev/null +++ b/tests/general/test_bff_scoring.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +""" +Simple test to verify BestFirstCrawlingStrategy fixes. +This test crawls a real website and shows that: +1. Higher-scoring pages are crawled first (priority queue fix) +2. Links are scored before truncation (link discovery fix) +""" + +import asyncio +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig +from crawl4ai.deep_crawling import BestFirstCrawlingStrategy +from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer + +async def test_best_first_strategy(): + """Test BestFirstCrawlingStrategy with keyword scoring""" + + print("=" * 70) + print("Testing BestFirstCrawlingStrategy with Real URL") + print("=" * 70) + print("\nThis test will:") + print("1. Crawl Python.org documentation") + print("2. Score pages based on keywords: 'tutorial', 'guide', 'reference'") + print("3. Show that higher-scoring pages are crawled first") + print("-" * 70) + + # Create a keyword scorer that prioritizes tutorial/guide pages + scorer = KeywordRelevanceScorer( + keywords=["tutorial", "guide", "reference", "documentation"], + weight=1.0, + case_sensitive=False + ) + + # Create the strategy with scoring + strategy = BestFirstCrawlingStrategy( + max_depth=2, # Crawl 2 levels deep + max_pages=10, # Limit to 10 pages total + url_scorer=scorer, # Use keyword scoring + include_external=False # Only internal links + ) + + # Configure browser and crawler + browser_config = BrowserConfig( + headless=True, # Run in background + verbose=False # Reduce output noise + ) + + crawler_config = CrawlerRunConfig( + deep_crawl_strategy=strategy, + verbose=False + ) + + print("\nStarting crawl of https://docs.python.org/3/") + print("Looking for pages with keywords: tutorial, guide, reference, documentation") + print("-" * 70) + + crawled_urls = [] + + async with AsyncWebCrawler(config=browser_config) as crawler: + # Crawl and collect results + results = await crawler.arun( + url="https://docs.python.org/3/", + config=crawler_config + ) + + # Process results + if isinstance(results, list): + for result in results: + score = result.metadata.get('score', 0) if result.metadata else 0 + depth = result.metadata.get('depth', 0) if result.metadata else 0 + crawled_urls.append({ + 'url': result.url, + 'score': score, + 'depth': depth, + 'success': result.success + }) + + print("\n" + "=" * 70) + print("CRAWL RESULTS (in order of crawling)") + print("=" * 70) + + for i, item in enumerate(crawled_urls, 1): + status = "✓" if item['success'] else "✗" + # Highlight high-scoring pages + if item['score'] > 0.5: + print(f"{i:2}. [{status}] Score: {item['score']:.2f} | Depth: {item['depth']} | {item['url']}") + print(f" ^ HIGH SCORE - Contains keywords!") + else: + print(f"{i:2}. [{status}] Score: {item['score']:.2f} | Depth: {item['depth']} | {item['url']}") + + print("\n" + "=" * 70) + print("ANALYSIS") + print("=" * 70) + + # Check if higher scores appear early in the crawl + scores = [item['score'] for item in crawled_urls[1:]] # Skip initial URL + high_score_indices = [i for i, s in enumerate(scores) if s > 0.3] + + if high_score_indices and high_score_indices[0] < len(scores) / 2: + print("✅ SUCCESS: Higher-scoring pages (with keywords) were crawled early!") + print(" This confirms the priority queue fix is working.") + else: + print("⚠️ Check the crawl order above - higher scores should appear early") + + # Show score distribution + print(f"\nScore Statistics:") + print(f" - Total pages crawled: {len(crawled_urls)}") + print(f" - Average score: {sum(item['score'] for item in crawled_urls) / len(crawled_urls):.2f}") + print(f" - Max score: {max(item['score'] for item in crawled_urls):.2f}") + print(f" - Pages with keywords: {sum(1 for item in crawled_urls if item['score'] > 0.3)}") + + print("\n" + "=" * 70) + print("TEST COMPLETE") + print("=" * 70) + +if __name__ == "__main__": + print("\n🔍 BestFirstCrawlingStrategy Simple Test\n") + asyncio.run(test_best_first_strategy()) \ No newline at end of file From ecbe5ffb84a769492863b6a602f013dfaa920ce7 Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Wed, 13 Aug 2025 18:16:46 +0530 Subject: [PATCH 005/119] docs: Update URL seeding examples to use proper async context managers - Wrap all AsyncUrlSeeder usage with async context managers - Update URL seeding adventure example to use "sitemap+cc" source, focus on course posts, and add stream=True parameter to fix runtime error --- docs/md_v2/core/url-seeding.md | 64 ++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/docs/md_v2/core/url-seeding.md b/docs/md_v2/core/url-seeding.md index f891c204..106a80a0 100644 --- a/docs/md_v2/core/url-seeding.md +++ b/docs/md_v2/core/url-seeding.md @@ -102,16 +102,16 @@ async def smart_blog_crawler(): # Step 2: Configure discovery - let's find all blog posts config = SeedingConfig( - source="sitemap", # Use the website's sitemap - pattern="*/blog/*.html", # Only blog posts + source="sitemap+cc", # Use the website's sitemap+cc + pattern="*/courses/*", # Only courses related posts extract_head=True, # Get page metadata max_urls=100 # Limit for this example ) # Step 3: Discover URLs from the Python blog - print("🔍 Discovering blog posts...") + print("🔍 Discovering course posts...") urls = await seeder.urls("realpython.com", config) - print(f"✅ Found {len(urls)} blog posts") + print(f"✅ Found {len(urls)} course posts") # Step 4: Filter for Python tutorials (using metadata!) tutorials = [ @@ -134,7 +134,8 @@ async def smart_blog_crawler(): async with AsyncWebCrawler() as crawler: config = CrawlerRunConfig( only_text=True, - word_count_threshold=300 # Only substantial articles + word_count_threshold=300, # Only substantial articles + stream=True ) # Extract URLs and crawl them @@ -155,7 +156,7 @@ asyncio.run(smart_blog_crawler()) **What just happened?** -1. We discovered all blog URLs from the sitemap +1. We discovered all blog URLs from the sitemap+cc 2. We filtered using metadata (no crawling needed!) 3. We crawled only the relevant tutorials 4. We saved tons of time and bandwidth @@ -282,8 +283,8 @@ config = SeedingConfig( live_check=True, # Verify each URL is accessible concurrency=20 # Check 20 URLs in parallel ) - -urls = await seeder.urls("example.com", config) +async with AsyncUrlSeeder() as seeder: + urls = await seeder.urls("example.com", config) # Now you can filter by status live_urls = [u for u in urls if u["status"] == "valid"] @@ -311,8 +312,8 @@ This is where URL seeding gets really powerful. Instead of crawling entire pages config = SeedingConfig( extract_head=True # Extract metadata from section ) - -urls = await seeder.urls("example.com", config) +async with AsyncUrlSeeder() as seeder: + urls = await seeder.urls("example.com", config) # Now each URL has rich metadata for url in urls[:3]: @@ -387,8 +388,8 @@ config = SeedingConfig( scoring_method="bm25", score_threshold=0.3 ) - -urls = await seeder.urls("example.com", config) +async with AsyncUrlSeeder() as seeder: + urls = await seeder.urls("example.com", config) # URLs are scored based on: # 1. Domain parts matching (e.g., 'python' in python.example.com) @@ -429,8 +430,8 @@ config = SeedingConfig( extract_head=True, live_check=True ) - -urls = await seeder.urls("blog.example.com", config) +async with AsyncUrlSeeder() as seeder: + urls = await seeder.urls("blog.example.com", config) # Analyze the results for url in urls[:5]: @@ -488,8 +489,8 @@ config = SeedingConfig( scoring_method="bm25", # Use BM25 algorithm score_threshold=0.3 # Minimum relevance score ) - -urls = await seeder.urls("realpython.com", config) +async with AsyncUrlSeeder() as seeder: + urls = await seeder.urls("realpython.com", config) # Results are automatically sorted by relevance! for url in urls[:5]: @@ -511,8 +512,8 @@ config = SeedingConfig( score_threshold=0.5, max_urls=20 ) - -urls = await seeder.urls("docs.example.com", config) +async with AsyncUrlSeeder() as seeder: + urls = await seeder.urls("docs.example.com", config) # The highest scoring URLs will be API docs! ``` @@ -529,8 +530,8 @@ config = SeedingConfig( score_threshold=0.4, pattern="*/product/*" # Combine with pattern matching ) - -urls = await seeder.urls("shop.example.com", config) +async with AsyncUrlSeeder() as seeder: + urls = await seeder.urls("shop.example.com", config) # Filter further by price (from metadata) affordable = [ @@ -550,8 +551,8 @@ config = SeedingConfig( scoring_method="bm25", score_threshold=0.35 ) - -urls = await seeder.urls("technews.com", config) +async with AsyncUrlSeeder() as seeder: + urls = await seeder.urls("technews.com", config) # Filter by date from datetime import datetime, timedelta @@ -591,8 +592,8 @@ for query in queries: score_threshold=0.4, max_urls=10 # Top 10 per topic ) - - urls = await seeder.urls("learning-platform.com", config) + async with AsyncUrlSeeder() as seeder: + urls = await seeder.urls("learning-platform.com", config) all_tutorials.extend(urls) # Remove duplicates while preserving order @@ -625,7 +626,8 @@ config = SeedingConfig( ) # Returns a dictionary: {domain: [urls]} -results = await seeder.many_urls(domains, config) +async with AsyncUrlSeeder() as seeder: + results = await seeder.many_urls(domains, config) # Process results for domain, urls in results.items(): @@ -654,8 +656,8 @@ config = SeedingConfig( pattern="*/blog/*", max_urls=100 ) - -results = await seeder.many_urls(competitors, config) +async with AsyncUrlSeeder() as seeder: + results = await seeder.many_urls(competitors, config) # Analyze content types for domain, urls in results.items(): @@ -690,8 +692,8 @@ config = SeedingConfig( score_threshold=0.3, max_urls=20 # Per site ) - -results = await seeder.many_urls(educational_sites, config) +async with AsyncUrlSeeder() as seeder: + results = await seeder.many_urls(educational_sites, config) # Find the best beginner tutorials all_tutorials = [] @@ -731,8 +733,8 @@ config = SeedingConfig( score_threshold=0.5, # High threshold for relevance max_urls=10 ) - -results = await seeder.many_urls(news_sites, config) +async with AsyncUrlSeeder() as seeder: + results = await seeder.many_urls(news_sites, config) # Collect all mentions mentions = [] From f4a432829effa4ef798760a127ef162cd8d3f4d4 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 18 Aug 2025 10:59:14 +0800 Subject: [PATCH 006/119] fix(crawler): Removed the incorrect reference in browser_config variable #1310 --- crawl4ai/async_crawler_strategy.back.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawl4ai/async_crawler_strategy.back.py b/crawl4ai/async_crawler_strategy.back.py index 9fdb0fe2..9f1ed38d 100644 --- a/crawl4ai/async_crawler_strategy.back.py +++ b/crawl4ai/async_crawler_strategy.back.py @@ -824,7 +824,7 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy): except Error: visibility_info = await self.check_visibility(page) - if self.browser_config.config.verbose: + if self.browser_config.verbose: self.logger.debug( message="Body visibility info: {info}", tag="DEBUG", From 9447054a654530ad7966e68c42d56f9a4c14643c Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 18 Aug 2025 14:20:05 +0800 Subject: [PATCH 007/119] docs: update Docker instructions to use the latest release tag --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 16fa42a1..131d55f3 100644 --- a/README.md +++ b/README.md @@ -304,9 +304,9 @@ The new Docker implementation includes: ### Getting Started ```bash -# Pull and run the latest release candidate -docker pull unclecode/crawl4ai:0.7.0 -docker run -d -p 11235:11235 --name crawl4ai --shm-size=1g unclecode/crawl4ai:0.7.0 +# Pull and run the latest release +docker pull unclecode/crawl4ai:latest +docker run -d -p 11235:11235 --name crawl4ai --shm-size=1g unclecode/crawl4ai:latest # Visit the playground at http://localhost:11235/playground ``` From 95051020f4f1888ebb431805e9b36168c8ddb7e0 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Thu, 21 Aug 2025 14:01:04 +0800 Subject: [PATCH 008/119] fix(docker): Fix LLM API key handling for multi-provider support Previously, the system incorrectly used OPENAI_API_KEY for all LLM providers due to a hardcoded api_key_env fallback in config.yml. This caused authentication errors when using non-OpenAI providers like Gemini. Changes: - Remove api_key_env from config.yml to let litellm handle provider-specific env vars - Simplify get_llm_api_key() to return None, allowing litellm to auto-detect keys - Update validate_llm_provider() to trust litellm's built-in key detection - Update documentation to reflect the new automatic key handling The fix leverages litellm's existing capability to automatically find the correct environment variable for each provider (OPENAI_API_KEY, GEMINI_API_TOKEN, etc.) without manual configuration. ref #1291 --- deploy/docker/README.md | 3 +-- deploy/docker/api.py | 6 +++--- deploy/docker/config.yml | 3 +-- deploy/docker/utils.py | 29 ++++++++++------------------ docs/md_v2/core/docker-deployment.md | 5 ++--- 5 files changed, 17 insertions(+), 29 deletions(-) diff --git a/deploy/docker/README.md b/deploy/docker/README.md index 49e0030b..d35050cc 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -692,8 +692,7 @@ app: # Default LLM Configuration llm: provider: "openai/gpt-4o-mini" # Can be overridden by LLM_PROVIDER env var - api_key_env: "OPENAI_API_KEY" - # api_key: sk-... # If you pass the API key directly then api_key_env will be ignored + # api_key: sk-... # If you pass the API key directly (not recommended) # Redis Configuration (Used by internal Redis server managed by supervisord) redis: diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 627d0bf4..c01c5ca7 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -96,7 +96,7 @@ async def handle_llm_qa( response = perform_completion_with_backoff( provider=config["llm"]["provider"], prompt_with_variables=prompt, - api_token=get_llm_api_key(config) + api_token=get_llm_api_key(config) # Returns None to let litellm handle it ) return response.choices[0].message.content @@ -127,7 +127,7 @@ async def process_llm_extraction( "error": error_msg }) return - api_key = get_llm_api_key(config, provider) + api_key = get_llm_api_key(config, provider) # Returns None to let litellm handle it llm_strategy = LLMExtractionStrategy( llm_config=LLMConfig( provider=provider or config["llm"]["provider"], @@ -203,7 +203,7 @@ async def handle_markdown_request( FilterType.LLM: LLMContentFilter( llm_config=LLMConfig( provider=provider or config["llm"]["provider"], - api_token=get_llm_api_key(config, provider), + api_token=get_llm_api_key(config, provider), # Returns None to let litellm handle it ), instruction=query or "Extract main content" ) diff --git a/deploy/docker/config.yml b/deploy/docker/config.yml index c81badc4..f5046613 100644 --- a/deploy/docker/config.yml +++ b/deploy/docker/config.yml @@ -11,8 +11,7 @@ app: # Default LLM Configuration llm: provider: "openai/gpt-4o-mini" - api_key_env: "OPENAI_API_KEY" - # api_key: sk-... # If you pass the API key directly then api_key_env will be ignored + # api_key: sk-... # If you pass the API key directly (not recommended) # Redis Configuration redis: diff --git a/deploy/docker/utils.py b/deploy/docker/utils.py index 2e2a80ac..8ec591e5 100644 --- a/deploy/docker/utils.py +++ b/deploy/docker/utils.py @@ -71,7 +71,7 @@ def decode_redis_hash(hash_data: Dict[bytes, bytes]) -> Dict[str, str]: -def get_llm_api_key(config: Dict, provider: Optional[str] = None) -> str: +def get_llm_api_key(config: Dict, provider: Optional[str] = None) -> Optional[str]: """Get the appropriate API key based on the LLM provider. Args: @@ -79,19 +79,14 @@ def get_llm_api_key(config: Dict, provider: Optional[str] = None) -> str: provider: Optional provider override (e.g., "openai/gpt-4") Returns: - The API key for the provider, or empty string if not found + The API key if directly configured, otherwise None to let litellm handle it """ - - # Use provided provider or fall back to config - if not provider: - provider = config["llm"]["provider"] - - # Check if direct API key is configured + # Check if direct API key is configured (for backward compatibility) if "api_key" in config["llm"]: return config["llm"]["api_key"] - # Fall back to the configured api_key_env if no match - return os.environ.get(config["llm"].get("api_key_env", ""), "") + # Return None - litellm will automatically find the right environment variable + return None def validate_llm_provider(config: Dict, provider: Optional[str] = None) -> tuple[bool, str]: @@ -104,16 +99,12 @@ def validate_llm_provider(config: Dict, provider: Optional[str] = None) -> tuple Returns: Tuple of (is_valid, error_message) """ - # Use provided provider or fall back to config - if not provider: - provider = config["llm"]["provider"] - - # Get the API key for this provider - api_key = get_llm_api_key(config, provider) - - if not api_key: - return False, f"No API key found for provider '{provider}'. Please set the appropriate environment variable." + # If a direct API key is configured, validation passes + if "api_key" in config["llm"]: + return True, "" + # Otherwise, trust that litellm will find the appropriate environment variable + # We can't easily validate this without reimplementing litellm's logic return True, "" diff --git a/docs/md_v2/core/docker-deployment.md b/docs/md_v2/core/docker-deployment.md index 6e9a9704..deda8163 100644 --- a/docs/md_v2/core/docker-deployment.md +++ b/docs/md_v2/core/docker-deployment.md @@ -176,7 +176,7 @@ The Docker setup now supports flexible LLM provider configuration through three 3. **Config File Default**: Falls back to `config.yml` (default: `openai/gpt-4o-mini`) -The system automatically selects the appropriate API key based on the configured `api_key_env` in the config file. +The system automatically selects the appropriate API key based on the provider. LiteLLM handles finding the correct environment variable for each provider (e.g., OPENAI_API_KEY for OpenAI, GEMINI_API_TOKEN for Google Gemini, etc.). #### 3. Build and Run with Compose @@ -693,8 +693,7 @@ app: # Default LLM Configuration llm: provider: "openai/gpt-4o-mini" # Can be overridden by LLM_PROVIDER env var - api_key_env: "OPENAI_API_KEY" - # api_key: sk-... # If you pass the API key directly then api_key_env will be ignored + # api_key: sk-... # If you pass the API key directly (not recommended) # Redis Configuration (Used by internal Redis server managed by supervisord) redis: From c09a57644f2b210f3af945cace6565890c892b27 Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Thu, 21 Aug 2025 19:11:31 +0530 Subject: [PATCH 009/119] =?UTF-8?q?docs:=20update=20adaptive=20crawler=20d?= =?UTF-8?q?ocs=20and=20cache=20defaults;=20remove=20deprecated=20examples?= =?UTF-8?q?=20(#1330)=20-=20Replace=20BaseStrategy=20with=20CrawlStrategy?= =?UTF-8?q?=20in=20custom=20strategy=20examples=20(DomainSpecificStrategy,?= =?UTF-8?q?=20HybridStrategy)=20-=20Remove=20=E2=80=9CCustom=20Link=20Scor?= =?UTF-8?q?ing=E2=80=9D=20and=20=E2=80=9CCaching=20Strategy=E2=80=9D=20sec?= =?UTF-8?q?tions=20no=20longer=20aligned=20with=20current=20library=20-=20?= =?UTF-8?q?Revise=20memory=20pruning=20example=20to=20use=20adaptive.get?= =?UTF-8?q?=5Frelevant=5Fcontent=20and=20index-based=20retention=20of=20to?= =?UTF-8?q?p=20500=20docs=20-=20Correct=20Quickstart=20note:=20default=20c?= =?UTF-8?q?ache=20mode=20is=20CacheMode.BYPASS;=20instruct=20enabling=20wi?= =?UTF-8?q?th=20CacheMode.ENABLED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/md_v2/advanced/adaptive-strategies.md | 50 ++++------------------ docs/md_v2/core/quickstart.md | 2 +- 2 files changed, 10 insertions(+), 42 deletions(-) diff --git a/docs/md_v2/advanced/adaptive-strategies.md b/docs/md_v2/advanced/adaptive-strategies.md index 4ab5b4cd..11c5585a 100644 --- a/docs/md_v2/advanced/adaptive-strategies.md +++ b/docs/md_v2/advanced/adaptive-strategies.md @@ -126,30 +126,6 @@ Factors: - URL depth (fewer slashes = higher authority) - Clean URL structure -### Custom Link Scoring - -```python -class CustomLinkScorer: - def score(self, link: Link, query: str, state: CrawlState) -> float: - # Prioritize specific URL patterns - if "/api/reference/" in link.href: - return 2.0 # Double the score - - # Deprioritize certain sections - if "/archive/" in link.href: - return 0.1 # Reduce score by 90% - - # Default scoring - return 1.0 - -# Use with adaptive crawler -adaptive = AdaptiveCrawler( - crawler, - config=config, - link_scorer=CustomLinkScorer() -) -``` - ## Domain-Specific Configurations ### Technical Documentation @@ -230,8 +206,12 @@ config = AdaptiveConfig( # Periodically clean state if len(state.knowledge_base) > 1000: - # Keep only most relevant - state.knowledge_base = get_top_relevant(state.knowledge_base, 500) + # Keep only the top 500 most relevant docs + top_content = adaptive.get_relevant_content(top_k=500) + keep_indices = {d["index"] for d in top_content} + state.knowledge_base = [ + doc for i, doc in enumerate(state.knowledge_base) if i in keep_indices + ] ``` ### Parallel Processing @@ -252,18 +232,6 @@ tasks = [ results = await asyncio.gather(*tasks) ``` -### Caching Strategy - -```python -# Enable caching for repeated crawls -async with AsyncWebCrawler( - config=BrowserConfig( - cache_mode=CacheMode.ENABLED - ) -) as crawler: - adaptive = AdaptiveCrawler(crawler, config) -``` - ## Debugging & Analysis ### Enable Verbose Logging @@ -322,9 +290,9 @@ with open("crawl_analysis.json", "w") as f: ### Implementing a Custom Strategy ```python -from crawl4ai.adaptive_crawler import BaseStrategy +from crawl4ai.adaptive_crawler import CrawlStrategy -class DomainSpecificStrategy(BaseStrategy): +class DomainSpecificStrategy(CrawlStrategy): def calculate_coverage(self, state: CrawlState) -> float: # Custom coverage calculation # e.g., weight certain terms more heavily @@ -351,7 +319,7 @@ adaptive = AdaptiveCrawler( ### Combining Strategies ```python -class HybridStrategy(BaseStrategy): +class HybridStrategy(CrawlStrategy): def __init__(self): self.strategies = [ TechnicalDocStrategy(), diff --git a/docs/md_v2/core/quickstart.md b/docs/md_v2/core/quickstart.md index e9a4b987..83cb6cef 100644 --- a/docs/md_v2/core/quickstart.md +++ b/docs/md_v2/core/quickstart.md @@ -79,7 +79,7 @@ if __name__ == "__main__": asyncio.run(main()) ``` -> IMPORTANT: By default cache mode is set to `CacheMode.ENABLED`. So to have fresh content, you need to set it to `CacheMode.BYPASS` +> IMPORTANT: By default cache mode is set to `CacheMode.BYPASS` to have fresh content. Set `CacheMode.ENABLED` to enable caching. We’ll explore more advanced config in later tutorials (like enabling proxies, PDF output, multi-tab sessions, etc.). For now, just note how you pass these objects to manage crawling. From 40ab287c9087bf6701072558427d974510335bc0 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Fri, 22 Aug 2025 12:05:21 +0800 Subject: [PATCH 010/119] fix(utils): Improve URL normalization by avoiding quote/unquote to preserve '+' signs. ref #1332 --- crawl4ai/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py index 73f1d2a3..09e6e4b7 100644 --- a/crawl4ai/utils.py +++ b/crawl4ai/utils.py @@ -2184,8 +2184,10 @@ def normalize_url( netloc = parsed.netloc.lower() # ── path ── - # Strip duplicate slashes and trailing “/” (except root) - path = quote(unquote(parsed.path)) + # Strip duplicate slashes and trailing "/" (except root) + # IMPORTANT: Don't use quote(unquote()) as it mangles + signs in URLs + # The path from urlparse is already properly encoded + path = parsed.path if path.endswith('/') and path != '/': path = path.rstrip('/') From b1dff5a4d3715886c0381d07e72cec391363e9f7 Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Sun, 24 Aug 2025 18:20:15 +0530 Subject: [PATCH 011/119] feat: Add comprehensive website to API example with frontend This commit adds a complete, web scraping API example that demonstrates how to get structured data from any website and use it like an API using the crawl4ai library with a minimalist frontend interface. Core Functionality - AI-powered web scraping with plain English queries - Dual scraping approaches: Schema-based (faster) and LLM-based (flexible) - Intelligent schema caching for improved performance - Custom LLM model support with API key management - Automatic duplicate request prevention Modern Frontend Interface - Minimalist black-and-white design inspired by modern web apps - Responsive layout with smooth animations and transitions - Three main pages: Scrape Data, Models Management, API Request History - Real-time results display with JSON formatting - Copy-to-clipboard functionality for extracted data - Toast notifications for user feedback - Auto-scroll to results when scraping starts Model Management System - Web-based model configuration interface - Support for any LLM provider (OpenAI, Gemini, Anthropic, etc.) - Simplified configuration requiring only provider and API token - Add, list, and delete model configurations - Secure storage of API keys in local JSON files API Request History - Automatic saving of all API requests and responses - Display of request history with URL, query, and cURL commands - Duplicate prevention (same URL + query combinations) - Request deletion functionality - Clean, simplified display focusing on essential information Technical Implementation Backend (FastAPI) - RESTful API with comprehensive endpoints - Pydantic models for request/response validation - Async web scraping with crawl4ai library - Error handling with detailed error messages - File-based storage for models and request history Frontend (Vanilla JS/CSS/HTML) - No framework dependencies - pure HTML, CSS, JavaScript - Modern CSS Grid and Flexbox layouts - Custom dropdown styling with SVG arrows - Responsive design for mobile and desktop - Smooth scrolling and animations Core Library Integration - WebScraperAgent class for orchestration - ModelConfig class for LLM configuration management - Schema generation and caching system - LLM extraction strategy support - Browser configuration with headless mode --- docs/examples/website-to-api/.gitignore | 221 +++++ docs/examples/website-to-api/README.md | 252 ++++++ docs/examples/website-to-api/api_server.py | 363 +++++++++ docs/examples/website-to-api/app.py | 49 ++ .../website-to-api/assets/crawl4ai_logo.jpg | Bin 0 -> 5920 bytes docs/examples/website-to-api/requirements.txt | 5 + .../examples/website-to-api/static/index.html | 201 +++++ docs/examples/website-to-api/static/script.js | 401 +++++++++ .../examples/website-to-api/static/styles.css | 765 ++++++++++++++++++ docs/examples/website-to-api/test_api.py | 28 + docs/examples/website-to-api/test_models.py | 67 ++ .../website-to-api/web_scraper_lib.py | 397 +++++++++ 12 files changed, 2749 insertions(+) create mode 100644 docs/examples/website-to-api/.gitignore create mode 100644 docs/examples/website-to-api/README.md create mode 100644 docs/examples/website-to-api/api_server.py create mode 100644 docs/examples/website-to-api/app.py create mode 100644 docs/examples/website-to-api/assets/crawl4ai_logo.jpg create mode 100644 docs/examples/website-to-api/requirements.txt create mode 100644 docs/examples/website-to-api/static/index.html create mode 100644 docs/examples/website-to-api/static/script.js create mode 100644 docs/examples/website-to-api/static/styles.css create mode 100644 docs/examples/website-to-api/test_api.py create mode 100644 docs/examples/website-to-api/test_models.py create mode 100644 docs/examples/website-to-api/web_scraper_lib.py diff --git a/docs/examples/website-to-api/.gitignore b/docs/examples/website-to-api/.gitignore new file mode 100644 index 00000000..8e884174 --- /dev/null +++ b/docs/examples/website-to-api/.gitignore @@ -0,0 +1,221 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock +#poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +#pdm.lock +#pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +#pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# Redis +*.rdb +*.aof +*.pid + +# RabbitMQ +mnesia/ +rabbitmq/ +rabbitmq-data/ + +# ActiveMQ +activemq-data/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# Streamlit +.streamlit/secrets.toml + +#directories +models +schemas +saved_requests \ No newline at end of file diff --git a/docs/examples/website-to-api/README.md b/docs/examples/website-to-api/README.md new file mode 100644 index 00000000..12ba4c4e --- /dev/null +++ b/docs/examples/website-to-api/README.md @@ -0,0 +1,252 @@ +# Web Scraper API with Custom Model Support + +A powerful web scraping API that converts any website into structured data using AI. Features a beautiful minimalist frontend interface and support for custom LLM models! + +## Features + +- **AI-Powered Scraping**: Provide a URL and plain English query to extract structured data +- **Beautiful Frontend**: Modern minimalist black-and-white interface with smooth UX +- **Custom Model Support**: Use any LLM provider (OpenAI, Gemini, Anthropic, etc.) with your own API keys +- **Model Management**: Save, list, and manage multiple model configurations via web interface +- **Dual Scraping Approaches**: Choose between Schema-based (faster) or LLM-based (more flexible) extraction +- **API Request History**: Automatic saving and display of all API requests with cURL commands +- **Schema Caching**: Intelligent caching of generated schemas for faster subsequent requests +- **Duplicate Prevention**: Avoids saving duplicate requests (same URL + query) +- **RESTful API**: Easy-to-use HTTP endpoints for all operations + +## Quick Start + +### 1. Install Dependencies + +```bash +pip install -r requirements.txt +``` + +### 2. Start the API Server + +```bash +python app.py +``` + +The server will start on `http://localhost:8000` with a beautiful web interface! + +### 3. Using the Web Interface + +Once the server is running, open your browser and go to `http://localhost:8000` to access the modern web interface! + +#### Pages: +- **Scrape Data**: Enter URLs and queries to extract structured data +- **Models**: Manage your AI model configurations (add, list, delete) +- **API Requests**: View history of all scraping requests with cURL commands + +#### Features: +- **Minimalist Design**: Clean black-and-white theme inspired by modern web apps +- **Real-time Results**: See extracted data in formatted JSON +- **Copy to Clipboard**: Easy copying of results +- **Toast Notifications**: User-friendly feedback +- **Dual Scraping Modes**: Choose between Schema-based and LLM-based approaches + +## Model Management + +### Adding Models via Web Interface + +1. Go to the **Models** page +2. Enter your model details: + - **Provider**: LLM provider (e.g., `gemini/gemini-2.5-flash`, `openai/gpt-4o`) + - **API Token**: Your API key for the provider +3. Click "Add Model" + +### API Usage for Model Management + +#### Save a Model Configuration + +```bash +curl -X POST "http://localhost:8000/models" \ + -H "Content-Type: application/json" \ + -d '{ + "provider": "gemini/gemini-2.5-flash", + "api_token": "your-api-key-here" + }' +``` + +#### List Saved Models + +```bash +curl -X GET "http://localhost:8000/models" +``` + +#### Delete a Model Configuration + +```bash +curl -X DELETE "http://localhost:8000/models/my-gemini" +``` + +## Scraping Approaches + +### 1. Schema-based Scraping (Faster) +- Generates CSS selectors for targeted extraction +- Caches schemas for repeated requests +- Faster execution for structured websites + +### 2. LLM-based Scraping (More Flexible) +- Direct LLM extraction without schema generation +- More flexible for complex or dynamic content +- Better for unstructured data extraction + +## Supported LLM Providers + +The API supports any LLM provider that crawl4ai supports, including: + +- **Google Gemini**: `gemini/gemini-2.5-flash`, `gemini/gemini-pro` +- **OpenAI**: `openai/gpt-4`, `openai/gpt-3.5-turbo` +- **Anthropic**: `anthropic/claude-3-opus`, `anthropic/claude-3-sonnet` +- **And more...** + +## API Endpoints + +### Core Endpoints + +- `POST /scrape` - Schema-based scraping +- `POST /scrape-with-llm` - LLM-based scraping +- `GET /schemas` - List cached schemas +- `POST /clear-cache` - Clear schema cache +- `GET /health` - Health check + +### Model Management Endpoints + +- `GET /models` - List saved model configurations +- `POST /models` - Save a new model configuration +- `DELETE /models/{model_name}` - Delete a model configuration + +### API Request History + +- `GET /saved-requests` - List all saved API requests +- `DELETE /saved-requests/{request_id}` - Delete a saved request + +## Request/Response Examples + +### Scrape Request + +```json +{ + "url": "https://example.com", + "query": "Extract the product name, price, and description", + "model_name": "my-custom-model" +} +``` + +### Scrape Response + +```json +{ + "success": true, + "url": "https://example.com", + "query": "Extract the product name, price, and description", + "extracted_data": { + "product_name": "Example Product", + "price": "$99.99", + "description": "This is an example product description" + }, + "schema_used": { ... }, + "timestamp": "2024-01-01T12:00:00Z" +} +``` + +### Model Configuration Request + +```json +{ + "provider": "gemini/gemini-2.5-flash", + "api_token": "your-api-key-here" +} +``` + +## Testing + +Run the test script to verify the model management functionality: + +```bash +python test_models.py +``` + +## File Structure + +``` +parse_example/ +├── api_server.py # FastAPI server with all endpoints +├── web_scraper_lib.py # Core scraping library +├── test_models.py # Test script for model management +├── requirements.txt # Dependencies +├── static/ # Frontend files +│ ├── index.html # Main HTML interface +│ ├── styles.css # CSS styles (minimalist theme) +│ └── script.js # JavaScript functionality +├── schemas/ # Cached schemas +├── models/ # Saved model configurations +├── saved_requests/ # API request history +└── README.md # This file +``` + +## Advanced Usage + +### Using the Library Directly + +```python +from web_scraper_lib import WebScraperAgent + +# Initialize agent +agent = WebScraperAgent() + +# Save a model configuration +agent.save_model_config( + model_name="my-model", + provider="openai/gpt-4", + api_token="your-api-key" +) + +# Schema-based scraping +result = await agent.scrape_data( + url="https://example.com", + query="Extract product information", + model_name="my-model" +) + +# LLM-based scraping +result = await agent.scrape_data_with_llm( + url="https://example.com", + query="Extract product information", + model_name="my-model" +) +``` + +### Schema Caching + +The system automatically caches generated schemas based on URL and query combinations: + +- **First request**: Generates schema using AI +- **Subsequent requests**: Uses cached schema for faster extraction + +### API Request History + +All API requests are automatically saved with: +- Request details (URL, query, model used) +- Response data +- Timestamp +- cURL command for re-execution + +### Duplicate Prevention + +The system prevents saving duplicate requests: +- Same URL + query combinations are not saved multiple times +- Returns existing request ID for duplicates +- Keeps the API request history clean + +## Error Handling + +The API provides detailed error messages for common issues: + +- Invalid URLs +- Missing model configurations +- API key errors +- Network timeouts +- Parsing errors diff --git a/docs/examples/website-to-api/api_server.py b/docs/examples/website-to-api/api_server.py new file mode 100644 index 00000000..0d4982ed --- /dev/null +++ b/docs/examples/website-to-api/api_server.py @@ -0,0 +1,363 @@ +from fastapi import FastAPI, HTTPException +from fastapi.staticfiles import StaticFiles +from fastapi.responses import FileResponse +from pydantic import BaseModel, HttpUrl +from typing import Dict, Any, Optional, Union, List +import uvicorn +import asyncio +import os +import json +from datetime import datetime +from web_scraper_lib import WebScraperAgent, scrape_website + +app = FastAPI( + title="Web Scraper API", + description="Convert any website into a structured data API. Provide a URL and tell AI what data you need in plain English.", + version="1.0.0" +) + +# Mount static files +if os.path.exists("static"): + app.mount("/static", StaticFiles(directory="static"), name="static") + +# Mount assets directory +if os.path.exists("assets"): + app.mount("/assets", StaticFiles(directory="assets"), name="assets") + +# Initialize the scraper agent +scraper_agent = WebScraperAgent() + +# Create directory for saved API requests +os.makedirs("saved_requests", exist_ok=True) + +class ScrapeRequest(BaseModel): + url: HttpUrl + query: str + model_name: Optional[str] = None + +class ModelConfigRequest(BaseModel): + model_name: str + provider: str + api_token: str + +class ScrapeResponse(BaseModel): + success: bool + url: str + query: str + extracted_data: Union[Dict[str, Any], list] + schema_used: Optional[Dict[str, Any]] = None + timestamp: Optional[str] = None + error: Optional[str] = None + +class SavedApiRequest(BaseModel): + id: str + endpoint: str + method: str + headers: Dict[str, str] + body: Dict[str, Any] + timestamp: str + response: Optional[Dict[str, Any]] = None + +def save_api_request(endpoint: str, method: str, headers: Dict[str, str], body: Dict[str, Any], response: Optional[Dict[str, Any]] = None) -> str: + """Save an API request to a JSON file.""" + + # Check for duplicate requests (same URL and query) + if endpoint in ["/scrape", "/scrape-with-llm"] and "url" in body and "query" in body: + existing_requests = get_saved_requests() + for existing_request in existing_requests: + if (existing_request.endpoint == endpoint and + existing_request.body.get("url") == body["url"] and + existing_request.body.get("query") == body["query"]): + print(f"Duplicate request found for URL: {body['url']} and query: {body['query']}") + return existing_request.id # Return existing request ID instead of creating new one + + request_id = datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3] + + saved_request = SavedApiRequest( + id=request_id, + endpoint=endpoint, + method=method, + headers=headers, + body=body, + timestamp=datetime.now().isoformat(), + response=response + ) + + file_path = os.path.join("saved_requests", f"{request_id}.json") + with open(file_path, "w") as f: + json.dump(saved_request.dict(), f, indent=2) + + return request_id + +def get_saved_requests() -> List[SavedApiRequest]: + """Get all saved API requests.""" + requests = [] + if os.path.exists("saved_requests"): + for filename in os.listdir("saved_requests"): + if filename.endswith('.json'): + file_path = os.path.join("saved_requests", filename) + try: + with open(file_path, "r") as f: + data = json.load(f) + requests.append(SavedApiRequest(**data)) + except Exception as e: + print(f"Error loading saved request {filename}: {e}") + + # Sort by timestamp (newest first) + requests.sort(key=lambda x: x.timestamp, reverse=True) + return requests + +@app.get("/") +async def root(): + """Serve the frontend interface.""" + if os.path.exists("static/index.html"): + return FileResponse("static/index.html") + else: + return { + "message": "Web Scraper API", + "description": "Convert any website into structured data with AI", + "endpoints": { + "/scrape": "POST - Scrape data from a website", + "/schemas": "GET - List cached schemas", + "/clear-cache": "POST - Clear schema cache", + "/models": "GET - List saved model configurations", + "/models": "POST - Save a new model configuration", + "/models/{model_name}": "DELETE - Delete a model configuration", + "/saved-requests": "GET - List saved API requests" + } + } + +@app.post("/scrape", response_model=ScrapeResponse) +async def scrape_website_endpoint(request: ScrapeRequest): + """ + Scrape structured data from any website. + + This endpoint: + 1. Takes a URL and plain English query + 2. Generates a custom scraper using AI + 3. Returns structured data + """ + try: + # Save the API request + headers = {"Content-Type": "application/json"} + body = { + "url": str(request.url), + "query": request.query, + "model_name": request.model_name + } + + result = await scraper_agent.scrape_data( + url=str(request.url), + query=request.query, + model_name=request.model_name + ) + + response_data = ScrapeResponse( + success=True, + url=result["url"], + query=result["query"], + extracted_data=result["extracted_data"], + schema_used=result["schema_used"], + timestamp=result["timestamp"] + ) + + # Save the request with response + save_api_request( + endpoint="/scrape", + method="POST", + headers=headers, + body=body, + response=response_data.dict() + ) + + return response_data + + except Exception as e: + # Save the failed request + headers = {"Content-Type": "application/json"} + body = { + "url": str(request.url), + "query": request.query, + "model_name": request.model_name + } + + save_api_request( + endpoint="/scrape", + method="POST", + headers=headers, + body=body, + response={"error": str(e)} + ) + + raise HTTPException(status_code=500, detail=f"Scraping failed: {str(e)}") + +@app.post("/scrape-with-llm", response_model=ScrapeResponse) +async def scrape_website_endpoint_with_llm(request: ScrapeRequest): + """ + Scrape structured data from any website using a custom LLM model. + """ + try: + # Save the API request + headers = {"Content-Type": "application/json"} + body = { + "url": str(request.url), + "query": request.query, + "model_name": request.model_name + } + + result = await scraper_agent.scrape_data_with_llm( + url=str(request.url), + query=request.query, + model_name=request.model_name + ) + + response_data = ScrapeResponse( + success=True, + url=result["url"], + query=result["query"], + extracted_data=result["extracted_data"], + timestamp=result["timestamp"] + ) + + # Save the request with response + save_api_request( + endpoint="/scrape-with-llm", + method="POST", + headers=headers, + body=body, + response=response_data.dict() + ) + + return response_data + + except Exception as e: + # Save the failed request + headers = {"Content-Type": "application/json"} + body = { + "url": str(request.url), + "query": request.query, + "model_name": request.model_name + } + + save_api_request( + endpoint="/scrape-with-llm", + method="POST", + headers=headers, + body=body, + response={"error": str(e)} + ) + + raise HTTPException(status_code=500, detail=f"Scraping failed: {str(e)}") + +@app.get("/saved-requests") +async def list_saved_requests(): + """List all saved API requests.""" + try: + requests = get_saved_requests() + return { + "success": True, + "requests": [req.dict() for req in requests], + "count": len(requests) + } + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to list saved requests: {str(e)}") + +@app.delete("/saved-requests/{request_id}") +async def delete_saved_request(request_id: str): + """Delete a saved API request.""" + try: + file_path = os.path.join("saved_requests", f"{request_id}.json") + if os.path.exists(file_path): + os.remove(file_path) + return { + "success": True, + "message": f"Saved request '{request_id}' deleted successfully" + } + else: + raise HTTPException(status_code=404, detail=f"Saved request '{request_id}' not found") + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to delete saved request: {str(e)}") + +@app.get("/schemas") +async def list_cached_schemas(): + """List all cached schemas.""" + try: + schemas = await scraper_agent.get_cached_schemas() + return { + "success": True, + "cached_schemas": schemas, + "count": len(schemas) + } + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to list schemas: {str(e)}") + +@app.post("/clear-cache") +async def clear_schema_cache(): + """Clear all cached schemas.""" + try: + scraper_agent.clear_cache() + return { + "success": True, + "message": "Schema cache cleared successfully" + } + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to clear cache: {str(e)}") + +@app.get("/models") +async def list_models(): + """List all saved model configurations.""" + try: + models = scraper_agent.list_saved_models() + return { + "success": True, + "models": models, + "count": len(models) + } + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to list models: {str(e)}") + +@app.post("/models") +async def save_model_config(request: ModelConfigRequest): + """Save a new model configuration.""" + try: + success = scraper_agent.save_model_config( + model_name=request.model_name, + provider=request.provider, + api_token=request.api_token + ) + + if success: + return { + "success": True, + "message": f"Model configuration '{request.model_name}' saved successfully" + } + else: + raise HTTPException(status_code=500, detail="Failed to save model configuration") + + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to save model: {str(e)}") + +@app.delete("/models/{model_name}") +async def delete_model_config(model_name: str): + """Delete a model configuration.""" + try: + success = scraper_agent.delete_model_config(model_name) + + if success: + return { + "success": True, + "message": f"Model configuration '{model_name}' deleted successfully" + } + else: + raise HTTPException(status_code=404, detail=f"Model configuration '{model_name}' not found") + + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to delete model: {str(e)}") + +@app.get("/health") +async def health_check(): + """Health check endpoint.""" + return {"status": "healthy", "service": "web-scraper-api"} + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=8000) \ No newline at end of file diff --git a/docs/examples/website-to-api/app.py b/docs/examples/website-to-api/app.py new file mode 100644 index 00000000..45710506 --- /dev/null +++ b/docs/examples/website-to-api/app.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +""" +Startup script for the Web Scraper API with frontend interface. +""" + +import os +import sys +import uvicorn +from pathlib import Path + +def main(): + # Check if static directory exists + static_dir = Path("static") + if not static_dir.exists(): + print("❌ Static directory not found!") + print("Please make sure the 'static' directory exists with the frontend files.") + sys.exit(1) + + # Check if required frontend files exist + required_files = ["index.html", "styles.css", "script.js"] + missing_files = [] + + for file in required_files: + if not (static_dir / file).exists(): + missing_files.append(file) + + if missing_files: + print(f"❌ Missing frontend files: {', '.join(missing_files)}") + print("Please make sure all frontend files are present in the static directory.") + sys.exit(1) + + print("🚀 Starting Web Scraper API with Frontend Interface") + print("=" * 50) + print("📁 Static files found and ready to serve") + print("🌐 Frontend will be available at: http://localhost:8000") + print("🔌 API endpoints available at: http://localhost:8000/docs") + print("=" * 50) + + # Start the server + uvicorn.run( + "api_server:app", + host="0.0.0.0", + port=8000, + reload=True, + log_level="info" + ) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/docs/examples/website-to-api/assets/crawl4ai_logo.jpg b/docs/examples/website-to-api/assets/crawl4ai_logo.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6a808c043126f1c691e7bbf81c766a4980b734cf GIT binary patch literal 5920 zcmb7IWmHvL*WM@SI&c8#JakBRN`rJsDi}1dl~q97#=0uqA4 zx4rki_xtmWcZ_eI^<$5*=UijW{XFZLbDrzD>#qPpQ&mG1fIt8M0{(#O1waXaL7{&Z zXmD_0;bLLI;aK=MIM}#^_=JQ6_yhz*#AFB}Vp3uP0t6+36iH4&K|x4zgNl-zij16s z{Ldy37#IV`!o$MCBPSvtBL9Eebq_#_3k(1QFbE3(C56C9A=iBX9e7CCUkmt;VdLOp zL7;FL_?8yT0}A_-%)bW5g2J$IAlIJ&LNE@BfFZz4heG~+|9?Js`S(kee5%JZUwOyg zw_s)M-jQ4por5)<5q~-4+_v~R?U`0dv(i$d2Ifvcyg2?iyTUdM3lJzfEO)!X0kL)7P$B^{6oe8yI20* zo!zCpE-eOOB5Snq{EggWvb_h|CzgZ01Xev;S6M}Tr9u;kHzDA8X}%>)n#frBr=zX` zUZ>$lC+k0|Bp4qWJFZtqPSJ|9%=k(%c4c~Zl+)d+-_acVDi{w2C8~N^Zx8E=d2Mr1 zFse31e+t}A;%ygL=IDPlhOXq#^i;S8G~R#6Tzp#SE%_&^hpeQ55G(lA+|_ z5BRG9ssn&PYGrUsB*RpcW<8<;bog&355mZf0H9DP$e@3rgk!<5Ay9yvLI4Lr%0x!# zh7<%5#RZXuU_r0M+F7@~j2aEq_((-E0R91u zSN=yg%h+nG>n1l;;2ti7$y)3+&(~L~rXBX8gN{;7_h3g`_It2vV9}Jz^KHmQg5$*j zg>Cm6*kPJ3<+SA&$g4XerG2BzJ|z4nu1hyLn;xT`SkEV*`a)DHCT;D-k)QYsKxZyn&MXb~){mex_fQ8E!(ZsGNw_{H$;%Ck@{Zi#yL+m~t z(-Y++E_KEEOW|ze^c>-Hyrv_SL&x)aIb3?RlpAKdYv((g>p_0`JslRw=#GRxAHoH0 z53KKFer_ZMs#!7eCBMCkg9=n(6B?h7R8(xGhQR-;<%BfqCJ4fkF{CUmLwMv22z?5p=fmKe~CX<~0KmY)V_ z4O^k@)_!kOE)Jp>$=pX%I_G^4$3YGWk^&GI6oQ9?^&dHaKw)qIi@ zQGAw%`Ob>cbFb@O{)E00L@Hw#>9uMg;>j-{K5w3HRdze&*eH}ltBzWCu!b1}iVRyZYTpElzz7n~Xx z8(-P0_D0rm)OF}GRdt*sz-OJlMOb2`TFGoC;fTPAmWWx$?BN3*N0Q^kYJo&Id#TwQ zFU()?xmgb}*jD>KsOp&V?DD?`xMv>=-$?0%>`YcC_K6*9yEAGwEF3GT`@Y;+Lgn?a zGTGK-a>Ll+mJFtMr?rEH3c30SAtkHrbRpYr{gsmo-thx-g}?f%Z}X$xGFz=B#{DX0 zR8Hxw+L!vc8F*yt;Vl3>UnI_i& z{>8-IwZe}&!rn2@jyD}y*vuEX3^bgNu67prCF=@Ae9ot<7avI`RYQ*P0n|Nd{*>Q* z)b^_kZ@I~vx&t}=qa3E>LRN@-tDW_e+__ClnuLa^6kO_R9W@%&627_Lc|Mufql`*w z*1$opTmvAQY^0zffPw&u`#<6VssW4?<*LWMW3>kt^!k{DmO&4}urvmp=UN{1wf? z?nl_ccvkoYUTsNPoJX3YSt@)=ftfC54<9u=KU9&GjuY1R5#uGVqrY3jq?z?nVTxT% zD3eHgav@hYJCCPTmjR{xY1@6(~#JU81nT}Zufi*keR zoh{O&yzZkiCHfCjonL)8kh-Qit0PkuF|T-PXXZ0q?j9kgWNM3sk|v={*gy)dFZdqu<0^K7gxQT zRx@T~nO0QRRGR14fEahp1gK^*<_`!}m&d2~J>DbC1Ijqemh z%XFCMTqbQVnok-{H%{Y}88v+xUG`OFq%00%)7_ul9fen12#C$UEkm8Tq&{o(77!gn zkB27|)1M0+7^Xb8((ITY`xF-u)OZumxR~CJ`5JJ+R`S%DQ(<^CsVkhCJi6PzU|NNG za@g>_6ay8` zwHRy%MR*y+Z&VP-7;yT1wlq3u?nvPI9+3S|2lHL5^&TJF?Y#<`r*f15=`CN&+Eh(6 z?NvEGxeN(6dQA|#_C4T`W*{kcb$yX!vmhBP_5lu^<(E?% z4qPQxX2_q1wdrC75*mztX6)f%@dG!pKigt`^FY_GPx9h5Q1xp_=MTcUStA79C|ACV z3%~rmA&n9!^QZe<&dYj3}0AAN|bAaf@)$?-$ZCNX`UmtUj~c;L?i#@QI}jm>5gpq%xm7O7mDr-w}$$oQW~(S|t~rsxml?h#(S z!Qyk2!Iac)&0w3@TbE1>i4{1Ihn5ssla*UNi{|wG^@(t_iN}U6c-4U;y=mRkU#`$w zfHvFLm6eC>HV15v|3uPWbZU6ia<6okkw0^CL^4UuX+w4aOW%zO_#0LNQRPW% zrb9>H9;2(>gud=o_kh|ZyC@YB+E?j1mzFmj9Ry>pf$Dgtdo8Qp1tp;+Ca4>bP4v*x zEBQn5?VtrOK5lf(d*{>=-nI`kuS2LSA-+irKtQ~;L!ifNd z6blN<%D?3iAY~TRSF}M0xc?E6?@V;9Mw<=j^4!{F&XAOiamNQ__RNkwB$;8@sIcNjH3Ufgkch!)9szL2gS_2Q{cjqVfj*}WZfv3fo_A{J?P zG&5CVlCSJ(JjFMM-JgMi>%!-J+wW)Ri)d$C$Yx8paQ5Hs6gaWL#5-`dUcpXg!$bBE z3@|&wk8wVXa>AnuL5)S-;(b6pc7;QtLkHahc{WP*Y|J~Z$@w3r%QXtJRnoFK=PW?R z91~2fGty~Q1|qt%o08*>Uu0HY8$yBo${)$q!GLY)@ecfy<~^a>x5b$z_TBRRIm>@* z?6VupID1clVk?kKRw#tt^^J!n_Iml8)1vAS;tYOVM!8rtI6}E^<^h_T+1qV*x0x0a) zEb(=|xnEx7n{Yg0HJHpck2KpeHEOYK^EMyfJpHkuvwr`})mA{#Z9kw!V=n0z9ZiN{ zllGg)8YTGF4>TtUmd=Y(D!HK7e3xBm|$aPFb5Wz$e7kFkj4;Se2pIQ`!Y zV_7K6VOz`QVA*A1rk|XxJ-26Houwbs7Go2l+xmK(+2pJJJ$^xz7#?!n%QCA-qk5P1 z!u*}~YZH{Yb@kSCP4sukTNt?w1{-SjCesiq;d=c?Jbp~Sh{sDJ-rN!Uv3u&Z{2(pG zGc=Ki2Pdd9qGL!SA!y-4ePI3E$8kKz&yP!;@EN?X0lc5urbf;8wG3kS+zPUh1;5_P zwtW=k7Mly&n}I?K$)G$9>(lRCd&{yv-oCn0<#pS(X~VWkc}+4x8UKDIx?d8lT^34+Es~xKSBkpW=>UC^-9mJr8Bk)D^8d&WDE9_q*CGe**BaX*q z{@dD2J07RL20*8=gZqnI9C4tx-0V@FDs2!Lc9f*WRo(d~{;LCW>NM`pUOZR_T(oz= z2U*c2nWOQJ|F%2m58i2ZQt+mNJuVL9-#bmptf&tN+PE|Qxzd>LbqKwG`&`5|kjNvF zfGSwqxIH0mgq#gR)1(W&dCxt>f_|Pw;&ZLw*&&q{ zQx7!;nB@Z<);hoSEqe~FmQ*ToHuzoiE|fND*Hi5_Gs-d~(;?(P2DwZ^Xnv$_E*K%CqMjY1jg*+3qjF=SrD8Iw*P|}T_f`9kFV-aw^oBAY$PIgLltseP4QQmH4ug65bfLC9+0oLwH#Qw1mQzlJ zLr7ek6|y_RJ^}yP02Nq~N#DjD97_;-&tCs$Ecw&aB2HfyFIURA zu`Z;q2NVcXD?xX7Hi`NJwb`80L`N2N9@tGh)@)kY5{{5MO|!BX!$kC@(=ooyuZ;22 zy1$&9$s4yzuejcau`+W~Y}QPYvm{U>b|0-`#@{b^|MI6Ho4V;G-A1O#1RSTk$FSyx zhWUv19yeW6)==v6)kqj14K|Fsn27kTk)xH0$HhPBC5981qX>}B&}To+UZQWhh;w2WPK;s0jI!8UUt-5CHRHa?8W~)1ZQxfu$v(kB!ys<(Ul?m0DO9 zDw!#TbQPf&?M+f|8Dp&EHEt6_^5E{?M?k@XF!Bs)dP_);dhr|H4?3;3vQbFFJ?Jzn z0Y8nn*oEn4>y-N>k7!RPnOi}GnS=+IQh9&(B<^fWl2BC`Yu#1#fr>iv0|%`h-}+}y zfsH-6m$<-+&ubvk*d7*?ku`H~Aa#I0+2-Dvs@nh{EJ>D4Z*b&*XmqK-Jk-&qBh92V z2Aq}EudoP|xPOUN0X58o%8tdZQ$!2U3zpVc#KRay-&~SbyiO^m=n^4TW-@k(hq%n< z;fUhgbvSB;&@HM6n-nLR=1ei<$-7OP)SF6D*zXKL$Y@DRtb(KxzAB9INg#RygQVeL|VX*Wly(A!iPMY@IRQ&s-K zxU?K#@jZ08esraSTIVKPfy+5LL;<44PQk3x`+F?m8y4*J>{mQ2Ho?m)8|`czc7yEF z4<1-J>Q)n$#T*HFPr#vJH!x7baXTInemK{lp0OSlFK$vX6_jQzI8&T?I4Q;==HaGs zuT+0EVV{?BR00-Xicisf4ItcLfc<(!M0l&TCP@m!uYWS0_3ksFcH<>pqCX(9$COeY zt5cFS_@IptkP$q3;o93}UpXMbb9iqaV6(YAZ=*I|#T^T0a*2^5A0gp)ycNp7g*%zd zb%K0{Ow{R#p<^!7#p+{6W%o2_rpI%78a|RvbU}on0}`z1aBTL9&cyWLWy%l*mpdST zQWu-aT1v$FYk>&x9J2@IQ_ipt5ve2@C9tYhov4@#-1jER+$K$0Y@@03$aq7oo3#>adFU zUmQ?AIn^`MyNp9^8p=ybONOPzYrgG9g|Fq0*(C@QBDqOr+y@%vtcwnE)O}twfAX^? zdhV?4ETEbN5_R*&y4S(Rn|G0O>+a}y~ + + + + + Web2API Example + + + + + +
+
+ + +
+
+ + +
+ +
+
+

Turn Any Website Into An API

+

This example shows how to turn any website into an API using Crawl4AI.

+
+ + +
+
+

1. Your Request

+
+
+ + +
+
+ + +
+
+
+ + +
+
+ + +
+
+ +
+
+ +
→
+ +
+

2. Your Instant API & Data

+
+
+ + 
curl -X POST http://localhost:8000/scrape -H "Content-Type: application/json" -d '{"url": "...", "query": "..."}'
+
+# Or for LLM-based approach:
+curl -X POST http://localhost:8000/scrape-with-llm -H "Content-Type: application/json" -d '{"url": "...", "query": "..."}'
+
+
+ + 
{
+  "success": true,
+  "extracted_data": [
+    {
+      "title": "Example Book",
+      "author": "John Doe",
+      "description": "A great book..."
+    }
+  ]
+}
+
+
+
+
+ + + + + + +
+ + +
+
+

Model Configuration

+

Configure and manage your AI model configurations

+
+ +
+ +
+

Add New Model

+
+
+
+ + +
+
+ + +
+
+ +
+ + +
+ + +
+
+ + +
+

Saved Models

+
+ +
+
+
+
+ + +
+
+

Saved API Requests

+

View and manage your previous API requests

+
+ +
+
+ +
+
+
+
+ + +
+ + + + \ No newline at end of file diff --git a/docs/examples/website-to-api/static/script.js b/docs/examples/website-to-api/static/script.js new file mode 100644 index 00000000..921598a7 --- /dev/null +++ b/docs/examples/website-to-api/static/script.js @@ -0,0 +1,401 @@ +// API Configuration +const API_BASE_URL = 'http://localhost:8000'; + +// DOM Elements +const navLinks = document.querySelectorAll('.nav-link'); +const pages = document.querySelectorAll('.page'); +const scrapeForm = document.getElementById('scrape-form'); +const modelForm = document.getElementById('model-form'); +const modelSelect = document.getElementById('model-select'); +const modelsList = document.getElementById('models-list'); +const resultsSection = document.getElementById('results-section'); +const loadingSection = document.getElementById('loading'); +const copyJsonBtn = document.getElementById('copy-json'); + +// Navigation +navLinks.forEach(link => { + link.addEventListener('click', (e) => { + e.preventDefault(); + const targetPage = link.dataset.page; + + // Update active nav link + navLinks.forEach(l => l.classList.remove('active')); + link.classList.add('active'); + + // Show target page + pages.forEach(page => page.classList.remove('active')); + document.getElementById(`${targetPage}-page`).classList.add('active'); + + // Load data for the page + if (targetPage === 'models') { + loadModels(); + } else if (targetPage === 'requests') { + loadSavedRequests(); + } + }); +}); + +// Scrape Form Handler +document.getElementById('extract-btn').addEventListener('click', async (e) => { + e.preventDefault(); + + // Scroll to results section immediately when button is clicked + document.getElementById('results-section').scrollIntoView({ + behavior: 'smooth', + block: 'start' + }); + + const url = document.getElementById('url').value; + const query = document.getElementById('query').value; + const headless = true; // Always use headless mode + const model_name = document.getElementById('model-select').value || null; + const scraping_approach = document.getElementById('scraping-approach').value; + + if (!url || !query) { + showToast('Please fill in both URL and query fields', 'error'); + return; + } + + if (!model_name) { + showToast('Please select a model from the dropdown or add one from the Models page', 'error'); + return; + } + + const data = { + url: url, + query: query, + headless: headless, + model_name: model_name + }; + + // Show loading state + showLoading(true); + hideResults(); + + try { + // Choose endpoint based on scraping approach + const endpoint = scraping_approach === 'llm' ? '/scrape-with-llm' : '/scrape'; + + const response = await fetch(`${API_BASE_URL}${endpoint}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(data) + }); + + const result = await response.json(); + + if (response.ok) { + displayResults(result); + showToast(`Data extracted successfully using ${scraping_approach === 'llm' ? 'LLM-based' : 'Schema-based'} approach!`, 'success'); + } else { + throw new Error(result.detail || 'Failed to extract data'); + } + } catch (error) { + console.error('Scraping error:', error); + showToast(`Error: ${error.message}`, 'error'); + } finally { + showLoading(false); + } +}); + +// Model Form Handler +modelForm.addEventListener('submit', async (e) => { + e.preventDefault(); + + const formData = new FormData(modelForm); + const data = { + model_name: formData.get('model_name'), + provider: formData.get('provider'), + api_token: formData.get('api_token') + }; + + try { + const response = await fetch(`${API_BASE_URL}/models`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(data) + }); + + const result = await response.json(); + + if (response.ok) { + showToast('Model saved successfully!', 'success'); + modelForm.reset(); + loadModels(); + loadModelSelect(); + } else { + throw new Error(result.detail || 'Failed to save model'); + } + } catch (error) { + console.error('Model save error:', error); + showToast(`Error: ${error.message}`, 'error'); + } +}); + +// Copy JSON Button +copyJsonBtn.addEventListener('click', () => { + const actualJsonOutput = document.getElementById('actual-json-output'); + const textToCopy = actualJsonOutput.textContent; + + navigator.clipboard.writeText(textToCopy).then(() => { + showToast('JSON copied to clipboard!', 'success'); + }).catch(() => { + showToast('Failed to copy JSON', 'error'); + }); +}); + +// Load Models +async function loadModels() { + try { + const response = await fetch(`${API_BASE_URL}/models`); + const result = await response.json(); + + if (response.ok) { + displayModels(result.models); + } else { + throw new Error(result.detail || 'Failed to load models'); + } + } catch (error) { + console.error('Load models error:', error); + showToast(`Error: ${error.message}`, 'error'); + } +} + +// Display Models +function displayModels(models) { + if (models.length === 0) { + modelsList.innerHTML = '

No models saved yet. Add your first model above!

'; + return; + } + + modelsList.innerHTML = models.map(model => ` +
+
+
${model}
+
Model Configuration
+
+
+ +
+
+ `).join(''); +} + +// Delete Model +async function deleteModel(modelName) { + if (!confirm(`Are you sure you want to delete the model "${modelName}"?`)) { + return; + } + + try { + const response = await fetch(`${API_BASE_URL}/models/${modelName}`, { + method: 'DELETE' + }); + + const result = await response.json(); + + if (response.ok) { + showToast('Model deleted successfully!', 'success'); + loadModels(); + loadModelSelect(); + } else { + throw new Error(result.detail || 'Failed to delete model'); + } + } catch (error) { + console.error('Delete model error:', error); + showToast(`Error: ${error.message}`, 'error'); + } +} + +// Load Model Select Options +async function loadModelSelect() { + try { + const response = await fetch(`${API_BASE_URL}/models`); + const result = await response.json(); + + if (response.ok) { + // Clear existing options + modelSelect.innerHTML = ''; + + // Add model options + result.models.forEach(model => { + const option = document.createElement('option'); + option.value = model; + option.textContent = model; + modelSelect.appendChild(option); + }); + } + } catch (error) { + console.error('Load model select error:', error); + } +} + +// Display Results +function displayResults(result) { + // Update result info + document.getElementById('result-url').textContent = result.url; + document.getElementById('result-query').textContent = result.query; + document.getElementById('result-model').textContent = result.model_name || 'Default Model'; + + // Display JSON in the actual results section + const actualJsonOutput = document.getElementById('actual-json-output'); + actualJsonOutput.textContent = JSON.stringify(result.extracted_data, null, 2); + + // Don't update the sample JSON in the workflow demo - keep it as example + + // Update the cURL example based on the approach used + const scraping_approach = document.getElementById('scraping-approach').value; + const endpoint = scraping_approach === 'llm' ? '/scrape-with-llm' : '/scrape'; + const curlExample = document.getElementById('curl-example'); + curlExample.textContent = `curl -X POST http://localhost:8000${endpoint} -H "Content-Type: application/json" -d '{"url": "${result.url}", "query": "${result.query}"}'`; + + // Show results section + resultsSection.style.display = 'block'; + resultsSection.scrollIntoView({ behavior: 'smooth' }); +} + +// Show/Hide Loading +function showLoading(show) { + loadingSection.style.display = show ? 'block' : 'none'; +} + +// Hide Results +function hideResults() { + resultsSection.style.display = 'none'; +} + +// Toast Notifications +function showToast(message, type = 'info') { + const toastContainer = document.getElementById('toast-container'); + const toast = document.createElement('div'); + toast.className = `toast ${type}`; + + const icon = type === 'success' ? 'fas fa-check-circle' : + type === 'error' ? 'fas fa-exclamation-circle' : + 'fas fa-info-circle'; + + toast.innerHTML = ` + + ${message} + `; + + toastContainer.appendChild(toast); + + // Auto remove after 5 seconds + setTimeout(() => { + toast.remove(); + }, 5000); +} + +// Load Saved Requests +async function loadSavedRequests() { + try { + const response = await fetch(`${API_BASE_URL}/saved-requests`); + const result = await response.json(); + + if (response.ok) { + displaySavedRequests(result.requests); + } else { + throw new Error(result.detail || 'Failed to load saved requests'); + } + } catch (error) { + console.error('Load saved requests error:', error); + showToast(`Error: ${error.message}`, 'error'); + } +} + +// Display Saved Requests +function displaySavedRequests(requests) { + const requestsList = document.getElementById('requests-list'); + + if (requests.length === 0) { + requestsList.innerHTML = '

No saved API requests yet. Make your first request from the Scrape page!

'; + return; + } + + requestsList.innerHTML = requests.map(request => { + const url = request.body.url; + const query = request.body.query; + const model = request.body.model_name || 'Default Model'; + const endpoint = request.endpoint; + + // Create curl command + const curlCommand = `curl -X POST http://localhost:8000${endpoint} \\ + -H "Content-Type: application/json" \\ + -d '{ + "url": "${url}", + "query": "${query}", + "model_name": "${model}" + }'`; + + return ` +
+
+
+
${url}
+
${query}
+
+
+ +
+
+ +
+

cURL Command:

+ 
${curlCommand}
+
+
+ `; + }).join(''); +} + +// Delete Saved Request +async function deleteSavedRequest(requestId) { + if (!confirm('Are you sure you want to delete this saved request?')) { + return; + } + + try { + const response = await fetch(`${API_BASE_URL}/saved-requests/${requestId}`, { + method: 'DELETE' + }); + + const result = await response.json(); + + if (response.ok) { + showToast('Saved request deleted successfully!', 'success'); + loadSavedRequests(); + } else { + throw new Error(result.detail || 'Failed to delete saved request'); + } + } catch (error) { + console.error('Delete saved request error:', error); + showToast(`Error: ${error.message}`, 'error'); + } +} + +// Initialize +document.addEventListener('DOMContentLoaded', () => { + loadModelSelect(); + + // Check if API is available + fetch(`${API_BASE_URL}/health`) + .then(response => { + if (!response.ok) { + showToast('Warning: API server might not be running', 'error'); + } + }) + .catch(() => { + showToast('Warning: Cannot connect to API server. Make sure it\'s running on localhost:8000', 'error'); + }); +}); \ No newline at end of file diff --git a/docs/examples/website-to-api/static/styles.css b/docs/examples/website-to-api/static/styles.css new file mode 100644 index 00000000..66d39760 --- /dev/null +++ b/docs/examples/website-to-api/static/styles.css @@ -0,0 +1,765 @@ +/* Reset and Base Styles */ +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + background: #000000; + color: #FFFFFF; + line-height: 1.6; + font-size: 16px; +} + +/* Header */ +.header { + border-bottom: 1px solid #333; + padding: 1rem 0; + background: #000000; + position: sticky; + top: 0; + z-index: 100; +} + +.header-content { + max-width: 1200px; + margin: 0 auto; + padding: 0 2rem; + display: flex; + justify-content: space-between; + align-items: center; +} + +.logo { + display: flex; + align-items: center; + gap: 0.5rem; + font-size: 1.5rem; + font-weight: 600; + color: #FFFFFF; +} + +.logo-image { + width: 40px; + height: 40px; + border-radius: 4px; + object-fit: contain; +} + +.nav-links { + display: flex; + gap: 2rem; +} + +.nav-link { + color: #CCCCCC; + text-decoration: none; + font-weight: 500; + transition: color 0.2s ease; +} + +.nav-link:hover, +.nav-link.active { + color: #FFFFFF; +} + +/* Main Content */ +.main-content { + max-width: 1200px; + margin: 0 auto; + padding: 2rem; +} + +.page { + display: none; +} + +.page.active { + display: block; +} + +/* Hero Section */ +.hero-section { + text-align: center; + margin-bottom: 4rem; + padding: 2rem 0; +} + +.hero-title { + font-size: 3rem; + font-weight: 700; + color: #FFFFFF; + margin-bottom: 1rem; + line-height: 1.2; +} + +.hero-subtitle { + font-size: 1.25rem; + color: #CCCCCC; + max-width: 600px; + margin: 0 auto; +} + +/* Workflow Demo */ +.workflow-demo { + display: grid; + grid-template-columns: 1fr auto 1fr; + gap: 2rem; + align-items: start; + margin-bottom: 4rem; +} + +.workflow-step { + display: flex; + flex-direction: column; + gap: 1rem; +} + +.step-title { + font-size: 1.25rem; + font-weight: 600; + color: #FFFFFF; + text-align: center; + margin-bottom: 1rem; +} + +.workflow-arrow { + font-size: 2rem; + font-weight: 700; + color: #09b5a5; + display: flex; + align-items: center; + justify-content: center; + margin-top: 20rem; +} + +/* Request Box */ +.request-box { + border: 2px solid #333; + border-radius: 8px; + padding: 2rem; + background: #111111; +} + +.input-group { + margin-bottom: 1.5rem; +} + +.input-group label { + display: block; + font-family: 'Courier New', monospace; + font-weight: 600; + color: #FFFFFF; + margin-bottom: 0.5rem; + font-size: 0.9rem; +} + +.input-group input, +.input-group textarea, +.input-group select { + width: 100%; + padding: 0.75rem; + border: 1px solid #333; + border-radius: 4px; + font-family: 'Courier New', monospace; + font-size: 0.9rem; + background: #1A1A1A; + color: #FFFFFF; + transition: border-color 0.2s ease; +} + +.input-group input:focus, +.input-group textarea:focus, +.input-group select:focus { + outline: none; + border-color: #09b5a5; +} + +.input-group textarea { + min-height: 80px; + resize: vertical; +} + +.form-options { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 1rem; + margin-bottom: 1.5rem; +} + +.option-group { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.option-group label { + font-family: 'Courier New', monospace; + font-weight: 600; + color: #FFFFFF; + font-size: 0.9rem; +} + +.option-group input[type="checkbox"] { + width: auto; + margin-right: 0.5rem; +} + +.extract-btn { + width: 100%; + padding: 1rem; + background: #09b5a5; + color: #000000; + border: none; + border-radius: 4px; + font-size: 1rem; + font-weight: 600; + cursor: pointer; + transition: background-color 0.2s ease; + display: flex; + align-items: center; + justify-content: center; + gap: 0.5rem; +} + +.extract-btn:hover { + background: #09b5a5; +} + +/* Dropdown specific styling */ +select, +.input-group select, +.option-group select { + cursor: pointer !important; + appearance: none !important; + -webkit-appearance: none !important; + -moz-appearance: none !important; + -ms-appearance: none !important; + background-image: url("data:image/svg+xml;charset=UTF-8,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='%23FFFFFF' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3e%3cpolyline points='6,9 12,15 18,9'%3e%3c/polyline%3e%3c/svg%3e") !important; + background-repeat: no-repeat !important; + background-position: right 0.75rem center !important; + background-size: 1rem !important; + padding-right: 2.5rem !important; + border: 1px solid #333 !important; + border-radius: 4px !important; + font-family: 'Courier New', monospace !important; + font-size: 0.9rem !important; + background-color: #1A1A1A !important; + color: #FFFFFF !important; +} + +select:hover, +.input-group select:hover, +.option-group select:hover { + border-color: #09b5a5 !important; +} + +select:focus, +.input-group select:focus, +.option-group select:focus { + outline: none !important; + border-color: #09b5a5 !important; +} + +select option, +.input-group select option, +.option-group select option { + background: #1A1A1A !important; + color: #FFFFFF !important; + padding: 0.5rem !important; +} + +/* Response Container */ +.response-container { + display: flex; + flex-direction: column; + gap: 1rem; +} + +.api-request-box, +.json-response-box { + border: 2px solid #333; + border-radius: 8px; + padding: 1.5rem; + background: #111111; +} + +.api-request-box label, +.json-response-box label { + display: block; + font-family: 'Courier New', monospace; + font-weight: 600; + color: #FFFFFF; + margin-bottom: 0.5rem; + font-size: 0.9rem; +} + +.api-request-box pre, +.json-response-box pre { + font-family: 'Courier New', monospace; + font-size: 0.85rem; + line-height: 1.5; + color: #FFFFFF; + background: #1A1A1A; + padding: 1rem; + border-radius: 4px; + overflow-x: auto; + white-space: pre-wrap; + word-break: break-all; +} + +/* Results Section */ +.results-section { + border: 2px solid #333; + border-radius: 8px; + overflow: hidden; + margin-top: 2rem; + background: #111111; +} + +.results-header { + background: #1A1A1A; + color: #FFFFFF; + padding: 1rem 1.5rem; + display: flex; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid #333; +} + +.results-header h2 { + font-size: 1.25rem; + font-weight: 600; + color: #FFFFFF; +} + +.copy-btn { + background: #09b5a5; + color: #000000; + border: none; + padding: 0.5rem 1rem; + border-radius: 4px; + font-size: 0.9rem; + font-weight: 600; + cursor: pointer; + display: flex; + align-items: center; + gap: 0.5rem; + transition: background-color 0.2s ease; +} + +.copy-btn:hover { + background: #09b5a5; +} + +.results-content { + padding: 1.5rem; +} + +.result-info { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + gap: 1rem; + margin-bottom: 1.5rem; + padding: 1rem; + background: #1A1A1A; + border-radius: 4px; + border: 1px solid #333; +} + +.info-item { + display: flex; + flex-direction: column; + gap: 0.25rem; +} + +.info-item .label { + font-weight: 600; + color: #FFFFFF; + font-size: 0.9rem; +} + +.info-item .value { + color: #CCCCCC; + word-break: break-all; +} + +.json-display { + background: #1A1A1A; + border-radius: 4px; + overflow: hidden; + border: 1px solid #333; +} + +.json-display pre { + color: #FFFFFF; + padding: 1.5rem; + margin: 0; + overflow-x: auto; + font-family: 'Courier New', monospace; + font-size: 0.9rem; + line-height: 1.5; +} + +/* Loading State */ +.loading { + text-align: center; + padding: 3rem; +} + +.spinner { + width: 40px; + height: 40px; + border: 3px solid #333; + border-top: 3px solid #09b5a5; + border-radius: 50%; + animation: spin 1s linear infinite; + margin: 0 auto 1rem; +} + +@keyframes spin { + 0% { transform: rotate(0deg); } + 100% { transform: rotate(360deg); } +} + +/* Models Page */ +.models-header { + text-align: center; + margin-bottom: 3rem; +} + +.models-header h1 { + font-size: 2.5rem; + font-weight: 700; + color: #FFFFFF; + margin-bottom: 1rem; +} + +.models-header p { + font-size: 1.1rem; + color: #CCCCCC; +} + +/* API Requests Page */ +.requests-header { + text-align: center; + margin-bottom: 3rem; +} + +.requests-header h1 { + font-size: 2.5rem; + font-weight: 700; + color: #FFFFFF; + margin-bottom: 1rem; +} + +.requests-header p { + font-size: 1.1rem; + color: #CCCCCC; +} + +.requests-container { + max-width: 1200px; + margin: 0 auto; +} + +.requests-list { + display: grid; + gap: 1.5rem; +} + +.request-card { + border: 2px solid #333; + border-radius: 8px; + padding: 1.5rem; + background: #111111; + transition: border-color 0.2s ease; +} + +.request-card:hover { + border-color: #09b5a5; +} + +.request-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 1rem; + padding-bottom: 1rem; + border-bottom: 1px solid #333; +} + +.request-info { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.request-url { + font-family: 'Courier New', monospace; + font-weight: 600; + color: #09b5a5; + font-size: 1.1rem; + word-break: break-all; +} + +.request-query { + color: #CCCCCC; + font-size: 0.9rem; + margin-top: 0.5rem; + word-break: break-all; +} + +.request-actions { + display: flex; + gap: 0.5rem; +} + +.request-curl { + background: #1A1A1A; + border: 1px solid #333; + border-radius: 4px; + padding: 1rem; + margin-top: 1rem; +} + +.request-curl h4 { + color: #FFFFFF; + font-size: 0.9rem; + font-weight: 600; + margin-bottom: 0.5rem; + font-family: 'Courier New', monospace; +} + +.request-curl pre { + color: #CCCCCC; + font-size: 0.8rem; + line-height: 1.4; + overflow-x: auto; + white-space: pre-wrap; + word-break: break-all; + background: #111111; + padding: 0.75rem; + border-radius: 4px; + border: 1px solid #333; +} + +.models-container { + max-width: 800px; + margin: 0 auto; +} + +.model-form-section { + border: 2px solid #333; + border-radius: 8px; + padding: 2rem; + margin-bottom: 2rem; + background: #111111; +} + +.model-form-section h3 { + font-size: 1.25rem; + font-weight: 600; + color: #FFFFFF; + margin-bottom: 1.5rem; +} + +.model-form { + display: flex; + flex-direction: column; + gap: 1.5rem; +} + +.form-row { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 1rem; +} + +.save-btn { + padding: 1rem; + background: #09b5a5; + color: #000000; + border: none; + border-radius: 4px; + font-size: 1rem; + font-weight: 600; + cursor: pointer; + transition: background-color 0.2s ease; + display: flex; + align-items: center; + justify-content: center; + gap: 0.5rem; +} + +.save-btn:hover { + background: #09b5a5; +} + +.saved-models-section h3 { + font-size: 1.25rem; + font-weight: 600; + color: #FFFFFF; + margin-bottom: 1.5rem; +} + +.models-list { + display: grid; + gap: 1rem; +} + +.model-card { + border: 2px solid #333; + border-radius: 8px; + padding: 1.5rem; + display: flex; + justify-content: space-between; + align-items: center; + transition: border-color 0.2s ease; + background: #111111; +} + +.model-card:hover { + border-color: #09b5a5; +} + +.model-info { + flex: 1; +} + +.model-name { + font-weight: 600; + color: #FFFFFF; + font-size: 1.1rem; + margin-bottom: 0.5rem; +} + +.model-provider { + color: #CCCCCC; + font-size: 0.9rem; +} + +.model-actions { + display: flex; + gap: 0.5rem; +} + +.btn-danger { + background: #FF4444; + color: #FFFFFF; + border: none; + padding: 0.5rem 1rem; + border-radius: 4px; + font-size: 0.9rem; + font-weight: 600; + cursor: pointer; + transition: background-color 0.2s ease; + display: flex; + align-items: center; + gap: 0.5rem; +} + +.btn-danger:hover { + background: #CC3333; +} + + + +/* Toast Notifications */ +.toast-container { + position: fixed; + top: 20px; + right: 20px; + z-index: 1000; +} + +.toast { + background: #111111; + border: 2px solid #333; + border-radius: 4px; + padding: 1rem 1.5rem; + margin-bottom: 0.5rem; + display: flex; + align-items: center; + gap: 0.5rem; + animation: slideIn 0.3s ease; + max-width: 400px; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3); + color: #FFFFFF; +} + +.toast.success { + border-color: #09b5a5; + background: #0A1A1A; +} + +.toast.error { + border-color: #FF4444; + background: #1A0A0A; +} + +.toast.info { + border-color: #09b5a5; + background: #0A1A1A; +} + +@keyframes slideIn { + from { + transform: translateX(100%); + opacity: 0; + } + to { + transform: translateX(0); + opacity: 1; + } +} + +/* Responsive Design */ +@media (max-width: 768px) { + .header-content { + padding: 0 1rem; + } + + .main-content { + padding: 1rem; + } + + .hero-title { + font-size: 2rem; + } + + .workflow-demo { + grid-template-columns: 1fr; + gap: 1rem; + } + + .workflow-arrow { + transform: rotate(90deg); + margin: 1rem 0; + } + + .form-options { + grid-template-columns: 1fr; + } + + .form-row { + grid-template-columns: 1fr; + } + + .result-info { + grid-template-columns: 1fr; + } + + .model-card { + flex-direction: column; + gap: 1rem; + text-align: center; + } + + .model-actions { + width: 100%; + justify-content: center; + } +} \ No newline at end of file diff --git a/docs/examples/website-to-api/test_api.py b/docs/examples/website-to-api/test_api.py new file mode 100644 index 00000000..8fd8db5f --- /dev/null +++ b/docs/examples/website-to-api/test_api.py @@ -0,0 +1,28 @@ +import asyncio +from web_scraper_lib import scrape_website +import os + +async def test_library(): + """Test the mini library directly.""" + print("=== Testing Mini Library ===") + + # Test 1: Scrape with a custom model + url = "https://marketplace.mainstreet.co.in/collections/adidas-yeezy/products/adidas-yeezy-boost-350-v2-yecheil-non-reflective" + query = "Extract the following data: Product name, Product price, Product description, Product size. DO NOT EXTRACT ANYTHING ELSE." + if os.path.exists("models"): + model_name = os.listdir("models")[0].split(".")[0] + else: + raise Exception("No models found in models directory") + + print(f"Scraping: {url}") + print(f"Query: {query}") + + try: + result = await scrape_website(url, query, model_name) + print("✅ Library test successful!") + print(f"Extracted data: {result['extracted_data']}") + except Exception as e: + print(f"❌ Library test failed: {e}") + +if __name__ == "__main__": + asyncio.run(test_library()) \ No newline at end of file diff --git a/docs/examples/website-to-api/test_models.py b/docs/examples/website-to-api/test_models.py new file mode 100644 index 00000000..2de0627e --- /dev/null +++ b/docs/examples/website-to-api/test_models.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +""" +Test script for the new model management functionality. +This script demonstrates how to save and use custom model configurations. +""" + +import asyncio +import requests +import json + +# API base URL +BASE_URL = "http://localhost:8000" + +def test_model_management(): + """Test the model management endpoints.""" + + print("=== Testing Model Management ===") + + # 1. List current models + print("\n1. Listing current models:") + response = requests.get(f"{BASE_URL}/models") + print(f"Status: {response.status_code}") + print(f"Response: {json.dumps(response.json(), indent=2)}") + + + # 2. Save another model configuration (OpenAI example) + print("\n2. Saving OpenAI model configuration:") + openai_config = { + "model_name": "my-openai", + "provider": "openai", + "api_token": "your-openai-api-key-here" + } + + response = requests.post(f"{BASE_URL}/models", json=openai_config) + print(f"Status: {response.status_code}") + print(f"Response: {json.dumps(response.json(), indent=2)}") + + # 3. List models again to see the new ones + print("\n3. Listing models after adding new ones:") + response = requests.get(f"{BASE_URL}/models") + print(f"Status: {response.status_code}") + print(f"Response: {json.dumps(response.json(), indent=2)}") + + # 4. Delete a model configuration + print("\n4. Deleting a model configuration:") + response = requests.delete(f"{BASE_URL}/models/my-openai") + print(f"Status: {response.status_code}") + print(f"Response: {json.dumps(response.json(), indent=2)}") + + # 5. Final list of models + print("\n5. Final list of models:") + response = requests.get(f"{BASE_URL}/models") + print(f"Status: {response.status_code}") + print(f"Response: {json.dumps(response.json(), indent=2)}") + +if __name__ == "__main__": + print("Model Management Test Script") + print("Make sure the API server is running on http://localhost:8000") + print("=" * 50) + + try: + test_model_management() + except requests.exceptions.ConnectionError: + print("Error: Could not connect to the API server.") + print("Make sure the server is running with: python api_server.py") + except Exception as e: + print(f"Error: {e}") \ No newline at end of file diff --git a/docs/examples/website-to-api/web_scraper_lib.py b/docs/examples/website-to-api/web_scraper_lib.py new file mode 100644 index 00000000..ded6f6f7 --- /dev/null +++ b/docs/examples/website-to-api/web_scraper_lib.py @@ -0,0 +1,397 @@ +from crawl4ai import ( + AsyncWebCrawler, + BrowserConfig, + CacheMode, + CrawlerRunConfig, + LLMConfig, + JsonCssExtractionStrategy, + LLMExtractionStrategy +) +import os +import json +import hashlib +from typing import Dict, Any, Optional, List +from litellm import completion + +class ModelConfig: + """Configuration for LLM models.""" + + def __init__(self, provider: str, api_token: str): + self.provider = provider + self.api_token = api_token + + def to_dict(self) -> Dict[str, Any]: + return { + "provider": self.provider, + "api_token": self.api_token + } + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> 'ModelConfig': + return cls( + provider=data["provider"], + api_token=data["api_token"] + ) + +class WebScraperAgent: + """ + A mini library that converts any website into a structured data API. + + Features: + 1. Provide a URL and tell AI what data you need in plain English + 2. Generate: Agent reverse-engineers the site and deploys custom scraper + 3. Integrate: Use private API endpoint to get structured data + 4. Support for custom LLM models and API keys + """ + + def __init__(self, schemas_dir: str = "schemas", models_dir: str = "models"): + self.schemas_dir = schemas_dir + self.models_dir = models_dir + os.makedirs(self.schemas_dir, exist_ok=True) + os.makedirs(self.models_dir, exist_ok=True) + + def _generate_schema_key(self, url: str, query: str) -> str: + """Generate a unique key for schema caching based on URL and query.""" + content = f"{url}:{query}" + return hashlib.md5(content.encode()).hexdigest() + + def save_model_config(self, model_name: str, provider: str, api_token: str) -> bool: + """ + Save a model configuration for later use. + + Args: + model_name: User-friendly name for the model + provider: LLM provider (e.g., 'gemini', 'openai', 'anthropic') + api_token: API token for the provider + + Returns: + True if saved successfully + """ + try: + model_config = ModelConfig(provider, api_token) + config_path = os.path.join(self.models_dir, f"{model_name}.json") + + with open(config_path, "w") as f: + json.dump(model_config.to_dict(), f, indent=2) + + print(f"Model configuration saved: {model_name}") + return True + except Exception as e: + print(f"Failed to save model configuration: {e}") + return False + + def load_model_config(self, model_name: str) -> Optional[ModelConfig]: + """ + Load a saved model configuration. + + Args: + model_name: Name of the saved model configuration + + Returns: + ModelConfig object or None if not found + """ + try: + config_path = os.path.join(self.models_dir, f"{model_name}.json") + if not os.path.exists(config_path): + return None + + with open(config_path, "r") as f: + data = json.load(f) + + return ModelConfig.from_dict(data) + except Exception as e: + print(f"Failed to load model configuration: {e}") + return None + + def list_saved_models(self) -> List[str]: + """List all saved model configurations.""" + models = [] + for filename in os.listdir(self.models_dir): + if filename.endswith('.json'): + models.append(filename[:-5]) # Remove .json extension + return models + + def delete_model_config(self, model_name: str) -> bool: + """ + Delete a saved model configuration. + + Args: + model_name: Name of the model configuration to delete + + Returns: + True if deleted successfully + """ + try: + config_path = os.path.join(self.models_dir, f"{model_name}.json") + if os.path.exists(config_path): + os.remove(config_path) + print(f"Model configuration deleted: {model_name}") + return True + return False + except Exception as e: + print(f"Failed to delete model configuration: {e}") + return False + + async def _load_or_generate_schema(self, url: str, query: str, session_id: str = "schema_generator", model_name: Optional[str] = None) -> Dict[str, Any]: + """ + Loads schema from cache if exists, otherwise generates using AI. + This is the "Generate" step - our agent reverse-engineers the site. + + Args: + url: URL to scrape + query: Query for data extraction + session_id: Session identifier + model_name: Name of saved model configuration to use + """ + schema_key = self._generate_schema_key(url, query) + schema_path = os.path.join(self.schemas_dir, f"{schema_key}.json") + + if os.path.exists(schema_path): + print(f"Schema found in cache for {url}") + with open(schema_path, "r") as f: + return json.load(f) + + print(f"Generating new schema for {url}") + print(f"Query: {query}") + query += """ + IMPORTANT: + GENERATE THE SCHEMA WITH ONLY THE FIELDS MENTIONED IN THE QUERY. MAKE SURE THE NUMBER OF FIELDS IN THE SCHEME MATCH THE NUMBER OF FIELDS IN THE QUERY. + """ + + # Step 1: Fetch the page HTML + async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler: + result = await crawler.arun( + url=url, + config=CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + session_id=session_id, + simulate_user=True, + remove_overlay_elements=True, + delay_before_return_html=5, + ) + ) + html = result.fit_html + + # Step 2: Generate schema using AI with custom model if specified + print("AI is analyzing the page structure...") + + # Use custom model configuration if provided + if model_name: + model_config = self.load_model_config(model_name) + if model_config: + llm_config = LLMConfig( + provider=model_config.provider, + api_token=model_config.api_token + ) + print(f"Using custom model: {model_name}") + else: + raise ValueError(f"Model configuration '{model_name}' not found. Please add it from the Models page.") + else: + # Require a model to be specified + raise ValueError("No model specified. Please select a model from the dropdown or add one from the Models page.") + + schema = JsonCssExtractionStrategy.generate_schema( + html=html, + llm_config=llm_config, + query=query + ) + + # Step 3: Cache the generated schema + print(f"Schema generated and cached: {json.dumps(schema, indent=2)}") + with open(schema_path, "w") as f: + json.dump(schema, f, indent=2) + + return schema + + def _generate_llm_schema(self, query: str, llm_config: LLMConfig) -> Dict[str, Any]: + """ + Generate a schema for a given query using a custom LLM model. + + Args: + query: Plain English description of what data to extract + model_config: Model configuration to use + """ + # ask the model to generate a schema for the given query in the form of a json. + prompt = f""" + IDENTIFY THE FIELDS FOR EXTRACTION MENTIONED IN THE QUERY and GENERATE A JSON SCHEMA FOR THE FIELDS. + eg. + {{ + "name": "str", + "age": "str", + "email": "str", + "product_name": "str", + "product_price": "str", + "product_description": "str", + "product_image": "str", + "product_url": "str", + "product_rating": "str", + "product_reviews": "str", + }} + Here is the query: + {query} + IMPORTANT: + THE RESULT SHOULD BE A JSON OBJECT. + MAKE SURE THE NUMBER OF FIELDS IN THE RESULT MATCH THE NUMBER OF FIELDS IN THE QUERY. + THE RESULT SHOULD BE A JSON OBJECT. + """ + response = completion( + model=llm_config.provider, + messages=[{"role": "user", "content": prompt}], + api_key=llm_config.api_token, + result_type="json" + ) + + return response.json()["choices"][0]["message"]["content"] + async def scrape_data_with_llm(self, url: str, query: str, model_name: Optional[str] = None) -> Dict[str, Any]: + """ + Scrape structured data from any website using a custom LLM model. + + Args: + url: The website URL to scrape + query: Plain English description of what data to extract + model_name: Name of saved model configuration to use + """ + + if model_name: + model_config = self.load_model_config(model_name) + if model_config: + llm_config = LLMConfig( + provider=model_config.provider, + api_token=model_config.api_token + ) + print(f"Using custom model: {model_name}") + else: + raise ValueError(f"Model configuration '{model_name}' not found. Please add it from the Models page.") + else: + # Require a model to be specified + raise ValueError("No model specified. Please select a model from the dropdown or add one from the Models page.") + + query += """\n + IMPORTANT: + THE RESULT SHOULD BE A JSON OBJECT WITH THE ONLY THE FIELDS MENTIONED IN THE QUERY. + MAKE SURE THE NUMBER OF FIELDS IN THE RESULT MATCH THE NUMBER OF FIELDS IN THE QUERY. + THE RESULT SHOULD BE A JSON OBJECT. + """ + + schema = self._generate_llm_schema(query, llm_config) + + print(f"Schema: {schema}") + + llm_extraction_strategy = LLMExtractionStrategy( + llm_config=llm_config, + instruction=query, + result_type="json", + schema=schema + ) + + async with AsyncWebCrawler() as crawler: + result = await crawler.arun( + url=url, + config=CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + simulate_user=True, + extraction_strategy=llm_extraction_strategy, + ) + ) + extracted_data = result.extracted_content + if isinstance(extracted_data, str): + try: + extracted_data = json.loads(extracted_data) + except json.JSONDecodeError: + # If it's not valid JSON, keep it as string + pass + + return { + "url": url, + "query": query, + "extracted_data": extracted_data, + "timestamp": result.timestamp if hasattr(result, 'timestamp') else None + } + + async def scrape_data(self, url: str, query: str, model_name: Optional[str] = None) -> Dict[str, Any]: + """ + Main method to scrape structured data from any website. + + Args: + url: The website URL to scrape + query: Plain English description of what data to extract + model_name: Name of saved model configuration to use + + Returns: + Structured data extracted from the website + """ + # Step 1: Generate or load schema (reverse-engineer the site) + schema = await self._load_or_generate_schema(url=url, query=query, model_name=model_name) + + # Step 2: Deploy custom high-speed scraper + print(f"Deploying custom scraper for {url}") + browser_config = BrowserConfig(headless=True) + + async with AsyncWebCrawler(config=browser_config) as crawler: + run_config = CrawlerRunConfig( + extraction_strategy=JsonCssExtractionStrategy(schema=schema), + ) + result = await crawler.arun(url=url, config=run_config) + + # Step 3: Return structured data + # Parse extracted_content if it's a JSON string + extracted_data = result.extracted_content + if isinstance(extracted_data, str): + try: + extracted_data = json.loads(extracted_data) + except json.JSONDecodeError: + # If it's not valid JSON, keep it as string + pass + + return { + "url": url, + "query": query, + "extracted_data": extracted_data, + "schema_used": schema, + "timestamp": result.timestamp if hasattr(result, 'timestamp') else None + } + + async def get_cached_schemas(self) -> Dict[str, str]: + """Get list of cached schemas.""" + schemas = {} + for filename in os.listdir(self.schemas_dir): + if filename.endswith('.json'): + schema_key = filename[:-5] # Remove .json extension + schemas[schema_key] = filename + return schemas + + def clear_cache(self): + """Clear all cached schemas.""" + import shutil + if os.path.exists(self.schemas_dir): + shutil.rmtree(self.schemas_dir) + os.makedirs(self.schemas_dir, exist_ok=True) + print("Schema cache cleared") + +# Convenience function for simple usage +async def scrape_website(url: str, query: str, model_name: Optional[str] = None) -> Dict[str, Any]: + """ + Simple function to scrape any website with plain English instructions. + + Args: + url: Website URL + query: Plain English description of what data to extract + model_name: Name of saved model configuration to use + + Returns: + Extracted structured data + """ + agent = WebScraperAgent() + return await agent.scrape_data(url, query, model_name) + +async def scrape_website_with_llm(url: str, query: str, model_name: Optional[str] = None): + """ + Scrape structured data from any website using a custom LLM model. + + Args: + url: The website URL to scrape + query: Plain English description of what data to extract + model_name: Name of saved model configuration to use + """ + agent = WebScraperAgent() + return await agent.scrape_data_with_llm(url, query, model_name) \ No newline at end of file From f2da460bb9a3e2a538ea6d4815799ccc5fffc2f7 Mon Sep 17 00:00:00 2001 From: "James T. Wood" Date: Sun, 24 Aug 2025 22:12:20 -0400 Subject: [PATCH 012/119] fix(dependencies): add cssselect to project dependencies Fixes bug reported in issue #1405 [Bug]: Excluded selector (excluded_selector) doesn't work This commit reintroduces the cssselect library which was removed by PR (https://github.com/unclecode/crawl4ai/pull/1368) and merged via (https://github.com/unclecode/crawl4ai/commit/437395e4902e2925ea55319353f64555b6da407b). Integration tested against 0.7.4 Docker container. Reintroducing cssselector package eliminated errors seen in logs and excluded_selector functionality was restored. Refs: #1405 --- pyproject.toml | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 9b00bd28..e7288dc2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ "PyYAML>=6.0", "nltk>=3.9.1", "rich>=13.9.4", + "cssselect>=1.2.0", "httpx>=0.27.2", "httpx[http2]>=0.27.2", "fake-useragent>=2.0.3", diff --git a/requirements.txt b/requirements.txt index 20f4df4f..0e66b3f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,6 +24,7 @@ psutil>=6.1.1 PyYAML>=6.0 nltk>=3.9.1 rich>=13.9.4 +cssselect>=1.2.0 chardet>=5.2.0 brotli>=1.1.0 httpx[http2]>=0.27.2 From 102352eac45c369f4df7fb6703c7ed84622b89dc Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 25 Aug 2025 14:04:08 +0800 Subject: [PATCH 013/119] fix(docker): resolve filter serialization and JSON encoding errors in deep crawl strategy (ref #1419) - Fix URLPatternFilter serialization by preventing private __slots__ from being serialized as constructor params - Add public attributes to URLPatternFilter to store original constructor parameters for proper serialization - Handle property descriptors in CrawlResult.model_dump() to prevent JSON serialization errors - Ensure filter chains work correctly with Docker client and REST API The issue occurred because: 1. Private implementation details (_simple_suffixes, etc.) were being serialized and passed as constructor arguments during deserialization 2. Property descriptors were being included in the serialized output, causing "Object of type property is not JSON serializable" errors Changes: - async_configs.py: Comment out __slots__ serialization logic (lines 100-109) - filters.py: Add patterns, use_glob, reverse to URLPatternFilter __slots__ and store as public attributes - models.py: Convert property descriptors to strings in model_dump() instead of including them directly --- crawl4ai/async_configs.py | 17 ++- crawl4ai/deep_crawling/filters.py | 8 + crawl4ai/models.py | 10 ++ tests/docker/test_filter_deep_crawl.py | 201 +++++++++++++++++++++++++ 4 files changed, 229 insertions(+), 7 deletions(-) create mode 100644 tests/docker/test_filter_deep_crawl.py diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index a43b50a4..0c843b2b 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -97,13 +97,16 @@ def to_serializable_dict(obj: Any, ignore_default_value : bool = False) -> Dict: if value != param.default and not ignore_default_value: current_values[name] = to_serializable_dict(value) - if hasattr(obj, '__slots__'): - for slot in obj.__slots__: - if slot.startswith('_'): # Handle private slots - attr_name = slot[1:] # Remove leading '_' - value = getattr(obj, slot, None) - if value is not None: - current_values[attr_name] = to_serializable_dict(value) + # Don't serialize private __slots__ - they're internal implementation details + # not constructor parameters. This was causing URLPatternFilter to fail + # because _simple_suffixes was being serialized as 'simple_suffixes' + # if hasattr(obj, '__slots__'): + # for slot in obj.__slots__: + # if slot.startswith('_'): # Handle private slots + # attr_name = slot[1:] # Remove leading '_' + # value = getattr(obj, slot, None) + # if value is not None: + # current_values[attr_name] = to_serializable_dict(value) diff --git a/crawl4ai/deep_crawling/filters.py b/crawl4ai/deep_crawling/filters.py index b65112e2..981cbcd8 100644 --- a/crawl4ai/deep_crawling/filters.py +++ b/crawl4ai/deep_crawling/filters.py @@ -120,6 +120,9 @@ class URLPatternFilter(URLFilter): """Pattern filter balancing speed and completeness""" __slots__ = ( + "patterns", # Store original patterns for serialization + "use_glob", # Store original use_glob for serialization + "reverse", # Store original reverse for serialization "_simple_suffixes", "_simple_prefixes", "_domain_patterns", @@ -142,6 +145,11 @@ class URLPatternFilter(URLFilter): reverse: bool = False, ): super().__init__() + # Store original constructor params for serialization + self.patterns = patterns + self.use_glob = use_glob + self.reverse = reverse + self._reverse = reverse patterns = [patterns] if isinstance(patterns, (str, Pattern)) else patterns diff --git a/crawl4ai/models.py b/crawl4ai/models.py index 640c2f2d..63e39885 100644 --- a/crawl4ai/models.py +++ b/crawl4ai/models.py @@ -253,6 +253,16 @@ class CrawlResult(BaseModel): requirements change, this is where you would update the logic. """ result = super().model_dump(*args, **kwargs) + + # Remove any property descriptors that might have been included + # These deprecated properties should not be in the serialized output + for key in ['fit_html', 'fit_markdown', 'markdown_v2']: + if key in result and isinstance(result[key], property): + # del result[key] + # Nasrin: I decided to convert it to string instead of removing it. + result[key] = str(result[key]) + + # Add the markdown field properly if self._markdown is not None: result["markdown"] = self._markdown.model_dump() return result diff --git a/tests/docker/test_filter_deep_crawl.py b/tests/docker/test_filter_deep_crawl.py new file mode 100644 index 00000000..4ee0df40 --- /dev/null +++ b/tests/docker/test_filter_deep_crawl.py @@ -0,0 +1,201 @@ +""" +Test the complete fix for both the filter serialization and JSON serialization issues. +""" + +import asyncio +import httpx + +from crawl4ai import BrowserConfig, CacheMode, CrawlerRunConfig +from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, FilterChain, URLPatternFilter + +BASE_URL = "http://localhost:11234/" # Adjust port as needed + +async def test_with_docker_client(): + """Test using the Docker client (same as 1419.py).""" + from crawl4ai.docker_client import Crawl4aiDockerClient + + print("=" * 60) + print("Testing with Docker Client") + print("=" * 60) + + try: + async with Crawl4aiDockerClient( + base_url=BASE_URL, + verbose=True, + ) as client: + + # Create filter chain - testing the serialization fix + filter_chain = [ + URLPatternFilter( + # patterns=["*about*", "*privacy*", "*terms*"], + patterns=["*advanced*"], + reverse=True + ), + ] + + crawler_config = CrawlerRunConfig( + deep_crawl_strategy=BFSDeepCrawlStrategy( + max_depth=2, # Keep it shallow for testing + # max_pages=5, # Limit pages for testing + filter_chain=FilterChain(filter_chain) + ), + cache_mode=CacheMode.BYPASS, + ) + + print("\n1. Testing crawl with filters...") + results = await client.crawl( + ["https://docs.crawl4ai.com"], # Simple test page + browser_config=BrowserConfig(headless=True), + crawler_config=crawler_config, + ) + + if results: + print(f"✅ Crawl succeeded! Type: {type(results)}") + if hasattr(results, 'success'): + print(f"✅ Results success: {results.success}") + # Test that we can iterate results without JSON errors + if hasattr(results, '__iter__'): + for i, result in enumerate(results): + if hasattr(result, 'url'): + print(f" Result {i}: {result.url[:50]}...") + else: + print(f" Result {i}: {str(result)[:50]}...") + else: + # Handle list of results + print(f"✅ Got {len(results)} results") + for i, result in enumerate(results[:3]): # Show first 3 + print(f" Result {i}: {result.url[:50]}...") + else: + print("❌ Crawl failed - no results returned") + return False + + print("\n✅ Docker client test completed successfully!") + return True + + except Exception as e: + print(f"❌ Docker client test failed: {e}") + import traceback + traceback.print_exc() + return False + + +async def test_with_rest_api(): + """Test using REST API directly.""" + print("\n" + "=" * 60) + print("Testing with REST API") + print("=" * 60) + + # Create filter configuration + deep_crawl_strategy_payload = { + "type": "BFSDeepCrawlStrategy", + "params": { + "max_depth": 2, + # "max_pages": 5, + "filter_chain": { + "type": "FilterChain", + "params": { + "filters": [ + { + "type": "URLPatternFilter", + "params": { + "patterns": ["*advanced*"], + "reverse": True + } + } + ] + } + } + } + } + + crawl_payload = { + "urls": ["https://docs.crawl4ai.com"], + "browser_config": {"type": "BrowserConfig", "params": {"headless": True}}, + "crawler_config": { + "type": "CrawlerRunConfig", + "params": { + "deep_crawl_strategy": deep_crawl_strategy_payload, + "cache_mode": "bypass" + } + } + } + + try: + async with httpx.AsyncClient() as client: + print("\n1. Sending crawl request to REST API...") + response = await client.post( + f"{BASE_URL}crawl", + json=crawl_payload, + timeout=30 + ) + + if response.status_code == 200: + print(f"✅ REST API returned 200 OK") + data = response.json() + if data.get("success"): + results = data.get("results", []) + print(f"✅ Got {len(results)} results") + for i, result in enumerate(results[:3]): + print(f" Result {i}: {result.get('url', 'unknown')[:50]}...") + else: + print(f"❌ Crawl not successful: {data}") + return False + else: + print(f"❌ REST API returned {response.status_code}") + print(f" Response: {response.text[:500]}") + return False + + print("\n✅ REST API test completed successfully!") + return True + + except Exception as e: + print(f"❌ REST API test failed: {e}") + import traceback + traceback.print_exc() + return False + + +async def main(): + """Run all tests.""" + print("\n🧪 TESTING COMPLETE FIX FOR DOCKER FILTER AND JSON ISSUES") + print("=" * 60) + print("Make sure the server is running with the updated code!") + print("=" * 60) + + results = [] + + # Test 1: Docker client + docker_passed = await test_with_docker_client() + results.append(("Docker Client", docker_passed)) + + # Test 2: REST API + rest_passed = await test_with_rest_api() + results.append(("REST API", rest_passed)) + + # Summary + print("\n" + "=" * 60) + print("FINAL TEST SUMMARY") + print("=" * 60) + + all_passed = True + for test_name, passed in results: + status = "✅ PASSED" if passed else "❌ FAILED" + print(f"{test_name:20} {status}") + if not passed: + all_passed = False + + print("=" * 60) + if all_passed: + print("🎉 ALL TESTS PASSED! Both issues are fully resolved!") + print("\nThe fixes:") + print("1. Filter serialization: Fixed by not serializing private __slots__") + print("2. JSON serialization: Fixed by removing property descriptors from model_dump()") + else: + print("⚠️ Some tests failed. Please check the server logs for details.") + + return 0 if all_passed else 1 + + +if __name__ == "__main__": + import sys + sys.exit(asyncio.run(main())) \ No newline at end of file From 38f3ea42a7d956dcca2c40bb6c86e1df778b6805 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Tue, 26 Aug 2025 12:06:56 +0800 Subject: [PATCH 014/119] fix(logger): ensure logger is a Logger instance in crawling strategies. ref #1437 --- crawl4ai/deep_crawling/bff_strategy.py | 8 +++++++- crawl4ai/deep_crawling/bfs_strategy.py | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.py index 7779c9f4..dc7a0bf0 100644 --- a/crawl4ai/deep_crawling/bff_strategy.py +++ b/crawl4ai/deep_crawling/bff_strategy.py @@ -47,7 +47,13 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy): self.url_scorer = url_scorer self.include_external = include_external self.max_pages = max_pages - self.logger = logger or logging.getLogger(__name__) + # self.logger = logger or logging.getLogger(__name__) + # Ensure logger is always a Logger instance, not a dict from serialization + if isinstance(logger, logging.Logger): + self.logger = logger + else: + # Create a new logger if logger is None, dict, or any other non-Logger type + self.logger = logging.getLogger(__name__) self.stats = TraversalStats(start_time=datetime.now()) self._cancel_event = asyncio.Event() self._pages_crawled = 0 diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.py index 950c3980..eb699f82 100644 --- a/crawl4ai/deep_crawling/bfs_strategy.py +++ b/crawl4ai/deep_crawling/bfs_strategy.py @@ -38,7 +38,13 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy): self.include_external = include_external self.score_threshold = score_threshold self.max_pages = max_pages - self.logger = logger or logging.getLogger(__name__) + # self.logger = logger or logging.getLogger(__name__) + # Ensure logger is always a Logger instance, not a dict from serialization + if isinstance(logger, logging.Logger): + self.logger = logger + else: + # Create a new logger if logger is None, dict, or any other non-Logger type + self.logger = logging.getLogger(__name__) self.stats = TraversalStats(start_time=datetime.now()) self._cancel_event = asyncio.Event() self._pages_crawled = 0 From 159207b86fc3920bd12192cbaab8374b2e7c6375 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Tue, 26 Aug 2025 16:44:07 +0800 Subject: [PATCH 015/119] feat(docker): Add temperature and base_url parameters for LLM configuration. ref #1035 Implement hierarchical configuration for LLM parameters with support for: - Temperature control (0.0-2.0) to adjust response creativity - Custom base_url for proxy servers and alternative endpoints - 4-tier priority: request params > provider env > global env > defaults Add helper functions in utils.py, update API schemas and handlers, support environment variables (LLM_TEMPERATURE, OPENAI_TEMPERATURE, etc.), and provide comprehensive documentation with examples. --- deploy/docker/.llm.env.example | 21 +- deploy/docker/api.py | 38 ++- deploy/docker/job.py | 6 +- deploy/docker/schemas.py | 2 + deploy/docker/server.py | 3 +- deploy/docker/utils.py | 63 +++++ docs/md_v2/core/docker-deployment.md | 144 ++++++++++- tests/docker/test_llm_params.py | 349 +++++++++++++++++++++++++++ 8 files changed, 603 insertions(+), 23 deletions(-) create mode 100755 tests/docker/test_llm_params.py diff --git a/deploy/docker/.llm.env.example b/deploy/docker/.llm.env.example index 254002f4..012435d8 100644 --- a/deploy/docker/.llm.env.example +++ b/deploy/docker/.llm.env.example @@ -10,4 +10,23 @@ GEMINI_API_TOKEN=your_gemini_key_here # Optional: Override the default LLM provider # Examples: "openai/gpt-4", "anthropic/claude-3-opus", "deepseek/chat", etc. # If not set, uses the provider specified in config.yml (default: openai/gpt-4o-mini) -# LLM_PROVIDER=anthropic/claude-3-opus \ No newline at end of file +# LLM_PROVIDER=anthropic/claude-3-opus + +# Optional: Global LLM temperature setting (0.0-2.0) +# Controls randomness in responses. Lower = more focused, Higher = more creative +# LLM_TEMPERATURE=0.7 + +# Optional: Global custom API base URL +# Use this to point to custom endpoints or proxy servers +# LLM_BASE_URL=https://api.custom.com/v1 + +# Optional: Provider-specific temperature overrides +# These take precedence over the global LLM_TEMPERATURE +# OPENAI_TEMPERATURE=0.5 +# ANTHROPIC_TEMPERATURE=0.3 +# GROQ_TEMPERATURE=0.8 + +# Optional: Provider-specific base URL overrides +# Use for provider-specific proxy endpoints +# OPENAI_BASE_URL=https://custom-openai.company.com/v1 +# GROQ_BASE_URL=https://custom-groq.company.com/v1 \ No newline at end of file diff --git a/deploy/docker/api.py b/deploy/docker/api.py index c01c5ca7..53359e1f 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -42,7 +42,9 @@ from utils import ( should_cleanup_task, decode_redis_hash, get_llm_api_key, - validate_llm_provider + validate_llm_provider, + get_llm_temperature, + get_llm_base_url ) import psutil, time @@ -96,7 +98,9 @@ async def handle_llm_qa( response = perform_completion_with_backoff( provider=config["llm"]["provider"], prompt_with_variables=prompt, - api_token=get_llm_api_key(config) # Returns None to let litellm handle it + api_token=get_llm_api_key(config), # Returns None to let litellm handle it + temperature=get_llm_temperature(config), + base_url=get_llm_base_url(config) ) return response.choices[0].message.content @@ -115,7 +119,9 @@ async def process_llm_extraction( instruction: str, schema: Optional[str] = None, cache: str = "0", - provider: Optional[str] = None + provider: Optional[str] = None, + temperature: Optional[float] = None, + base_url: Optional[str] = None ) -> None: """Process LLM extraction in background.""" try: @@ -131,7 +137,9 @@ async def process_llm_extraction( llm_strategy = LLMExtractionStrategy( llm_config=LLMConfig( provider=provider or config["llm"]["provider"], - api_token=api_key + api_token=api_key, + temperature=temperature or get_llm_temperature(config, provider), + base_url=base_url or get_llm_base_url(config, provider) ), instruction=instruction, schema=json.loads(schema) if schema else None, @@ -178,7 +186,9 @@ async def handle_markdown_request( query: Optional[str] = None, cache: str = "0", config: Optional[dict] = None, - provider: Optional[str] = None + provider: Optional[str] = None, + temperature: Optional[float] = None, + base_url: Optional[str] = None ) -> str: """Handle markdown generation requests.""" try: @@ -204,6 +214,8 @@ async def handle_markdown_request( llm_config=LLMConfig( provider=provider or config["llm"]["provider"], api_token=get_llm_api_key(config, provider), # Returns None to let litellm handle it + temperature=temperature or get_llm_temperature(config, provider), + base_url=base_url or get_llm_base_url(config, provider) ), instruction=query or "Extract main content" ) @@ -248,7 +260,9 @@ async def handle_llm_request( schema: Optional[str] = None, cache: str = "0", config: Optional[dict] = None, - provider: Optional[str] = None + provider: Optional[str] = None, + temperature: Optional[float] = None, + api_base_url: Optional[str] = None ) -> JSONResponse: """Handle LLM extraction requests.""" base_url = get_base_url(request) @@ -279,7 +293,9 @@ async def handle_llm_request( cache, base_url, config, - provider + provider, + temperature, + api_base_url ) except Exception as e: @@ -324,7 +340,9 @@ async def create_new_task( cache: str, base_url: str, config: dict, - provider: Optional[str] = None + provider: Optional[str] = None, + temperature: Optional[float] = None, + api_base_url: Optional[str] = None ) -> JSONResponse: """Create and initialize a new task.""" decoded_url = unquote(input_path) @@ -349,7 +367,9 @@ async def create_new_task( query, schema, cache, - provider + provider, + temperature, + api_base_url ) return JSONResponse({ diff --git a/deploy/docker/job.py b/deploy/docker/job.py index 10d83fdd..823dd8c8 100644 --- a/deploy/docker/job.py +++ b/deploy/docker/job.py @@ -37,6 +37,8 @@ class LlmJobPayload(BaseModel): schema: Optional[str] = None cache: bool = False provider: Optional[str] = None + temperature: Optional[float] = None + base_url: Optional[str] = None class CrawlJobPayload(BaseModel): @@ -63,6 +65,8 @@ async def llm_job_enqueue( cache=payload.cache, config=_config, provider=payload.provider, + temperature=payload.temperature, + api_base_url=payload.base_url, ) @@ -72,7 +76,7 @@ async def llm_job_status( task_id: str, _td: Dict = Depends(lambda: _token_dep()) ): - return await handle_task_status(_redis, task_id) + return await handle_task_status(_redis, task_id, base_url=str(request.base_url)) # ---------- CRAWL job ------------------------------------------------------- diff --git a/deploy/docker/schemas.py b/deploy/docker/schemas.py index 96196633..0d8335b6 100644 --- a/deploy/docker/schemas.py +++ b/deploy/docker/schemas.py @@ -16,6 +16,8 @@ class MarkdownRequest(BaseModel): q: Optional[str] = Field(None, description="Query string used by BM25/LLM filters") c: Optional[str] = Field("0", description="Cache‑bust / revision counter") provider: Optional[str] = Field(None, description="LLM provider override (e.g., 'anthropic/claude-3-opus')") + temperature: Optional[float] = Field(None, description="LLM temperature override (0.0-2.0)") + base_url: Optional[str] = Field(None, description="LLM API base URL override") class RawCode(BaseModel): diff --git a/deploy/docker/server.py b/deploy/docker/server.py index 57fd3d6d..e453758a 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -241,7 +241,8 @@ async def get_markdown( raise HTTPException( 400, "Invalid URL format. Must start with http://, https://, or for raw HTML (raw:, raw://)") markdown = await handle_markdown_request( - body.url, body.f, body.q, body.c, config, body.provider + body.url, body.f, body.q, body.c, config, body.provider, + body.temperature, body.base_url ) return JSONResponse({ "url": body.url, diff --git a/deploy/docker/utils.py b/deploy/docker/utils.py index 8ec591e5..5f3618af 100644 --- a/deploy/docker/utils.py +++ b/deploy/docker/utils.py @@ -108,6 +108,69 @@ def validate_llm_provider(config: Dict, provider: Optional[str] = None) -> tuple return True, "" +def get_llm_temperature(config: Dict, provider: Optional[str] = None) -> Optional[float]: + """Get temperature setting based on the LLM provider. + + Priority order: + 1. Provider-specific environment variable (e.g., OPENAI_TEMPERATURE) + 2. Global LLM_TEMPERATURE environment variable + 3. None (to use litellm/provider defaults) + + Args: + config: The application configuration dictionary + provider: Optional provider override (e.g., "openai/gpt-4") + + Returns: + The temperature setting if configured, otherwise None + """ + # Check provider-specific temperature first + if provider: + provider_name = provider.split('/')[0].upper() + provider_temp = os.environ.get(f"{provider_name}_TEMPERATURE") + if provider_temp: + try: + return float(provider_temp) + except ValueError: + logging.warning(f"Invalid temperature value for {provider_name}: {provider_temp}") + + # Check global LLM_TEMPERATURE + global_temp = os.environ.get("LLM_TEMPERATURE") + if global_temp: + try: + return float(global_temp) + except ValueError: + logging.warning(f"Invalid global temperature value: {global_temp}") + + # Return None to use litellm/provider defaults + return None + + +def get_llm_base_url(config: Dict, provider: Optional[str] = None) -> Optional[str]: + """Get base URL setting based on the LLM provider. + + Priority order: + 1. Provider-specific environment variable (e.g., OPENAI_BASE_URL) + 2. Global LLM_BASE_URL environment variable + 3. None (to use default endpoints) + + Args: + config: The application configuration dictionary + provider: Optional provider override (e.g., "openai/gpt-4") + + Returns: + The base URL if configured, otherwise None + """ + # Check provider-specific base URL first + if provider: + provider_name = provider.split('/')[0].upper() + provider_url = os.environ.get(f"{provider_name}_BASE_URL") + if provider_url: + return provider_url + + # Check global LLM_BASE_URL + return os.environ.get("LLM_BASE_URL") + + def verify_email_domain(email: str) -> bool: try: domain = email.split('@')[1] diff --git a/docs/md_v2/core/docker-deployment.md b/docs/md_v2/core/docker-deployment.md index deda8163..a98b7ab8 100644 --- a/docs/md_v2/core/docker-deployment.md +++ b/docs/md_v2/core/docker-deployment.md @@ -89,6 +89,16 @@ ANTHROPIC_API_KEY=your-anthropic-key # TOGETHER_API_KEY=your-together-key # MISTRAL_API_KEY=your-mistral-key # GEMINI_API_TOKEN=your-gemini-token + +# Optional: Global LLM settings +# LLM_PROVIDER=openai/gpt-4o-mini +# LLM_TEMPERATURE=0.7 +# LLM_BASE_URL=https://api.custom.com/v1 + +# Optional: Provider-specific overrides +# OPENAI_TEMPERATURE=0.5 +# OPENAI_BASE_URL=https://custom-openai.com/v1 +# ANTHROPIC_TEMPERATURE=0.3 EOL ``` > 🔑 **Note**: Keep your API keys secure! Never commit `.llm.env` to version control. @@ -156,28 +166,44 @@ cp deploy/docker/.llm.env.example .llm.env **Flexible LLM Provider Configuration:** -The Docker setup now supports flexible LLM provider configuration through three methods: +The Docker setup now supports flexible LLM provider configuration through a hierarchical system: -1. **Environment Variable** (Highest Priority): Set `LLM_PROVIDER` to override the default - ```bash - export LLM_PROVIDER="anthropic/claude-3-opus" - # Or in your .llm.env file: - # LLM_PROVIDER=anthropic/claude-3-opus - ``` - -2. **API Request Parameter**: Specify provider per request +1. **API Request Parameters** (Highest Priority): Specify per request ```json { "url": "https://example.com", "f": "llm", - "provider": "groq/mixtral-8x7b" + "provider": "groq/mixtral-8x7b", + "temperature": 0.7, + "base_url": "https://api.custom.com/v1" } ``` -3. **Config File Default**: Falls back to `config.yml` (default: `openai/gpt-4o-mini`) +2. **Provider-Specific Environment Variables**: Override for specific providers + ```bash + # In your .llm.env file: + OPENAI_TEMPERATURE=0.5 + OPENAI_BASE_URL=https://custom-openai.com/v1 + ANTHROPIC_TEMPERATURE=0.3 + ``` + +3. **Global Environment Variables**: Set defaults for all providers + ```bash + # In your .llm.env file: + LLM_PROVIDER=anthropic/claude-3-opus + LLM_TEMPERATURE=0.7 + LLM_BASE_URL=https://api.proxy.com/v1 + ``` + +4. **Config File Default**: Falls back to `config.yml` (default: `openai/gpt-4o-mini`) The system automatically selects the appropriate API key based on the provider. LiteLLM handles finding the correct environment variable for each provider (e.g., OPENAI_API_KEY for OpenAI, GEMINI_API_TOKEN for Google Gemini, etc.). +**Supported LLM Parameters:** +- `provider`: LLM provider and model (e.g., "openai/gpt-4", "anthropic/claude-3-opus") +- `temperature`: Controls randomness (0.0-2.0, lower = more focused, higher = more creative) +- `base_url`: Custom API endpoint for proxy servers or alternative endpoints + #### 3. Build and Run with Compose The `docker-compose.yml` file in the project root provides a simplified approach that automatically handles architecture detection using buildx. @@ -555,6 +581,101 @@ Crucially, when sending configurations directly via JSON, they **must** follow t **LLM Extraction Strategy** *(Keep example, ensure schema uses type/value wrapper)* *(Keep Deep Crawler Example)* +### LLM Configuration Examples + +The Docker API supports dynamic LLM configuration through multiple levels: + +#### Temperature Control + +Temperature affects the randomness of LLM responses (0.0 = deterministic, 2.0 = very creative): + +```python +import requests + +# Low temperature for factual extraction +response = requests.post( + "http://localhost:11235/md", + json={ + "url": "https://example.com", + "f": "llm", + "q": "Extract all dates and numbers from this page", + "temperature": 0.2 # Very focused, deterministic + } +) + +# High temperature for creative tasks +response = requests.post( + "http://localhost:11235/md", + json={ + "url": "https://example.com", + "f": "llm", + "q": "Write a creative summary of this content", + "temperature": 1.2 # More creative, varied responses + } +) +``` + +#### Custom API Endpoints + +Use custom base URLs for proxy servers or alternative API endpoints: + +```python + +# Using a local LLM server +response = requests.post( + "http://localhost:11235/md", + json={ + "url": "https://example.com", + "f": "llm", + "q": "Extract key information", + "provider": "ollama/llama2", + "base_url": "http://localhost:11434/v1" + } +) +``` + +#### Dynamic Provider Selection + +Switch between providers based on task requirements: + +```python +async def smart_extraction(url: str, content_type: str): + """Select provider and temperature based on content type""" + + configs = { + "technical": { + "provider": "openai/gpt-4", + "temperature": 0.3, + "query": "Extract technical specifications and code examples" + }, + "creative": { + "provider": "anthropic/claude-3-opus", + "temperature": 0.9, + "query": "Create an engaging narrative summary" + }, + "quick": { + "provider": "groq/mixtral-8x7b", + "temperature": 0.5, + "query": "Quick summary in bullet points" + } + } + + config = configs.get(content_type, configs["quick"]) + + response = await httpx.post( + "http://localhost:11235/md", + json={ + "url": url, + "f": "llm", + "q": config["query"], + "provider": config["provider"], + "temperature": config["temperature"] + } + ) + + return response.json() +``` + ### REST API Examples Update URLs to use port `11235`. @@ -694,6 +815,7 @@ app: llm: provider: "openai/gpt-4o-mini" # Can be overridden by LLM_PROVIDER env var # api_key: sk-... # If you pass the API key directly (not recommended) + # temperature and base_url are controlled via environment variables or request parameters # Redis Configuration (Used by internal Redis server managed by supervisord) redis: diff --git a/tests/docker/test_llm_params.py b/tests/docker/test_llm_params.py new file mode 100755 index 00000000..533c4482 --- /dev/null +++ b/tests/docker/test_llm_params.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +""" +Test script for LLM temperature and base_url parameters in Crawl4AI Docker API. +This demonstrates the new hierarchical configuration system: +1. Request-level parameters (highest priority) +2. Provider-specific environment variables +3. Global environment variables +4. System defaults (lowest priority) +""" + +import asyncio +import httpx +import json +import os +from rich.console import Console +from rich.panel import Panel +from rich.syntax import Syntax +from rich.table import Table + + +console = Console() + +# Configuration +BASE_URL = "http://localhost:11235" # Docker API endpoint +TEST_URL = "https://httpbin.org/html" # Simple test page + +# --- Helper Functions --- + +async def check_server_health(client: httpx.AsyncClient) -> bool: + """Check if the server is healthy.""" + console.print("[bold cyan]Checking server health...[/]", end="") + try: + response = await client.get("/health", timeout=10.0) + response.raise_for_status() + console.print(" [bold green]✓ Server is healthy![/]") + return True + except Exception as e: + console.print(f"\n[bold red]✗ Server health check failed: {e}[/]") + console.print(f"Is the server running at {BASE_URL}?") + return False + +def print_request(endpoint: str, payload: dict, title: str = "Request"): + """Pretty print the request.""" + syntax = Syntax(json.dumps(payload, indent=2), "json", theme="monokai") + console.print(Panel.fit( + f"[cyan]POST {endpoint}[/cyan]\n{syntax}", + title=f"[bold blue]{title}[/]", + border_style="blue" + )) + +def print_response(response: dict, title: str = "Response"): + """Pretty print relevant parts of the response.""" + # Extract only the relevant parts + relevant = {} + if "markdown" in response: + relevant["markdown"] = response["markdown"][:200] + "..." if len(response.get("markdown", "")) > 200 else response.get("markdown", "") + if "success" in response: + relevant["success"] = response["success"] + if "url" in response: + relevant["url"] = response["url"] + if "filter" in response: + relevant["filter"] = response["filter"] + + console.print(Panel.fit( + Syntax(json.dumps(relevant, indent=2), "json", theme="monokai"), + title=f"[bold green]{title}[/]", + border_style="green" + )) + +# --- Test Functions --- + +async def test_default_no_params(client: httpx.AsyncClient): + """Test 1: No temperature or base_url specified - uses defaults""" + console.rule("[bold yellow]Test 1: Default Configuration (No Parameters)[/]") + + payload = { + "url": TEST_URL, + "f": "llm", + "q": "What is the main heading of this page? Answer in exactly 5 words." + } + + print_request("/md", payload, "Request without temperature/base_url") + + try: + response = await client.post("/md", json=payload, timeout=30.0) + response.raise_for_status() + data = response.json() + print_response(data, "Response (using system defaults)") + console.print("[dim]→ This used system defaults or environment variables if set[/]") + except Exception as e: + console.print(f"[red]Error: {e}[/]") + +async def test_request_temperature(client: httpx.AsyncClient): + """Test 2: Request-level temperature (highest priority)""" + console.rule("[bold yellow]Test 2: Request-Level Temperature[/]") + + # Test with low temperature (more focused) + payload_low = { + "url": TEST_URL, + "f": "llm", + "q": "What is the main heading? Be creative and poetic.", + "temperature": 0.1 # Very low - should be less creative + } + + print_request("/md", payload_low, "Low Temperature (0.1)") + + try: + response = await client.post("/md", json=payload_low, timeout=30.0) + response.raise_for_status() + data_low = response.json() + print_response(data_low, "Response with Low Temperature") + console.print("[dim]→ Low temperature (0.1) should produce focused, less creative output[/]") + except Exception as e: + console.print(f"[red]Error: {e}[/]") + + console.print() + + # Test with high temperature (more creative) + payload_high = { + "url": TEST_URL, + "f": "llm", + "q": "What is the main heading? Be creative and poetic.", + "temperature": 1.5 # High - should be more creative + } + + print_request("/md", payload_high, "High Temperature (1.5)") + + try: + response = await client.post("/md", json=payload_high, timeout=30.0) + response.raise_for_status() + data_high = response.json() + print_response(data_high, "Response with High Temperature") + console.print("[dim]→ High temperature (1.5) should produce more creative, varied output[/]") + except Exception as e: + console.print(f"[red]Error: {e}[/]") + +async def test_provider_override(client: httpx.AsyncClient): + """Test 3: Provider override with temperature""" + console.rule("[bold yellow]Test 3: Provider Override with Temperature[/]") + + provider = "gemini/gemini-2.5-flash-lite" + payload = { + "url": TEST_URL, + "f": "llm", + "q": "Summarize this page in one sentence.", + "provider": provider, # Explicitly set provider + "temperature": 0.7 + } + + print_request("/md", payload, "Provider + Temperature Override") + + try: + response = await client.post("/md", json=payload, timeout=30.0) + response.raise_for_status() + data = response.json() + print_response(data, "Response with Provider Override") + console.print(f"[dim]→ This explicitly uses {provider} with temperature 0.7[/]") + except Exception as e: + console.print(f"[red]Error: {e}[/]") + +async def test_base_url_custom(client: httpx.AsyncClient): + """Test 4: Custom base_url (will fail unless you have a custom endpoint)""" + console.rule("[bold yellow]Test 4: Custom Base URL (Demo Only)[/]") + + payload = { + "url": TEST_URL, + "f": "llm", + "q": "What is this page about?", + "base_url": "https://api.custom-endpoint.com/v1", # Custom endpoint + "temperature": 0.5 + } + + print_request("/md", payload, "Custom Base URL Request") + console.print("[yellow]Note: This will fail unless you have a custom endpoint set up[/]") + + try: + response = await client.post("/md", json=payload, timeout=10.0) + response.raise_for_status() + data = response.json() + print_response(data, "Response from Custom Endpoint") + except httpx.HTTPStatusError as e: + console.print(f"[yellow]Expected failure (no custom endpoint): Status {e.response.status_code}[/]") + except Exception as e: + console.print(f"[yellow]Expected error: {e}[/]") + +async def test_llm_job_endpoint(client: httpx.AsyncClient): + """Test 5: Test the /llm/job endpoint with temperature and base_url""" + console.rule("[bold yellow]Test 5: LLM Job Endpoint with Parameters[/]") + + payload = { + "url": TEST_URL, + "q": "Extract the main title and any key information", + "temperature": 0.3, + # "base_url": "https://api.openai.com/v1" # Optional + } + + print_request("/llm/job", payload, "LLM Job with Temperature") + + try: + # Submit the job + response = await client.post("/llm/job", json=payload, timeout=30.0) + response.raise_for_status() + job_data = response.json() + + if "task_id" in job_data: + task_id = job_data["task_id"] + console.print(f"[green]Job created with task_id: {task_id}[/]") + + # Poll for result (simplified - in production use proper polling) + await asyncio.sleep(3) + + status_response = await client.get(f"/llm/job/{task_id}") + status_data = status_response.json() + + if status_data.get("status") == "completed": + console.print("[green]Job completed successfully![/]") + if "result" in status_data: + console.print(Panel.fit( + Syntax(json.dumps(status_data["result"], indent=2), "json", theme="monokai"), + title="Extraction Result", + border_style="green" + )) + else: + console.print(f"[yellow]Job status: {status_data.get('status', 'unknown')}[/]") + else: + console.print(f"[red]Unexpected response: {job_data}[/]") + + except Exception as e: + console.print(f"[red]Error: {e}[/]") + + +async def test_llm_endpoint(client: httpx.AsyncClient): + """ + Quick QA round-trip with /llm. + Asks a trivial question against SIMPLE_URL just to show wiring. + """ + import time + import urllib.parse + + page_url = "https://kidocode.com" + question = "What is the title of this page?" + + enc = urllib.parse.quote_plus(page_url, safe="") + console.print(f"GET /llm/{enc}?q={question}") + + try: + t0 = time.time() + resp = await client.get(f"/llm/{enc}", params={"q": question}) + dt = time.time() - t0 + console.print( + f"Response Status: [bold {'green' if resp.is_success else 'red'}]{resp.status_code}[/] (took {dt:.2f}s)") + resp.raise_for_status() + answer = resp.json().get("answer", "") + console.print(Panel(answer or "No answer returned", + title="LLM answer", border_style="magenta", expand=False)) + except Exception as e: + console.print(f"[bold red]Error hitting /llm:[/] {e}") + + +async def show_environment_info(): + """Display current environment configuration""" + console.rule("[bold cyan]Current Environment Configuration[/]") + + table = Table(title="LLM Environment Variables", show_header=True, header_style="bold magenta") + table.add_column("Variable", style="cyan", width=30) + table.add_column("Value", style="yellow") + table.add_column("Description", style="dim") + + env_vars = [ + ("LLM_PROVIDER", "Global default provider"), + ("LLM_TEMPERATURE", "Global default temperature"), + ("LLM_BASE_URL", "Global custom API endpoint"), + ("OPENAI_API_KEY", "OpenAI API key"), + ("OPENAI_TEMPERATURE", "OpenAI-specific temperature"), + ("OPENAI_BASE_URL", "OpenAI-specific endpoint"), + ("ANTHROPIC_API_KEY", "Anthropic API key"), + ("ANTHROPIC_TEMPERATURE", "Anthropic-specific temperature"), + ("GROQ_API_KEY", "Groq API key"), + ("GROQ_TEMPERATURE", "Groq-specific temperature"), + ] + + for var, desc in env_vars: + value = os.environ.get(var, "[not set]") + if "API_KEY" in var and value != "[not set]": + # Mask API keys for security + value = value[:10] + "..." if len(value) > 10 else "***" + table.add_row(var, value, desc) + + console.print(table) + console.print() + +# --- Main Test Runner --- + +async def main(): + """Run all tests""" + console.print(Panel.fit( + "[bold cyan]Crawl4AI LLM Parameters Test Suite[/]\n" + + "Testing temperature and base_url configuration hierarchy", + border_style="cyan" + )) + + # Show current environment + # await show_environment_info() + + # Create HTTP client + async with httpx.AsyncClient(base_url=BASE_URL, timeout=60.0) as client: + # Check server health + if not await check_server_health(client): + console.print("[red]Server is not available. Please ensure the Docker container is running.[/]") + return + + # Run tests + tests = [ + ("Default Configuration", test_default_no_params), + ("Request Temperature", test_request_temperature), + ("Provider Override", test_provider_override), + ("Custom Base URL", test_base_url_custom), + ("LLM Job Endpoint", test_llm_job_endpoint), + ("LLM Endpoint", test_llm_endpoint), + ] + + for i, (name, test_func) in enumerate(tests, 1): + if i > 1: + console.print() # Add spacing between tests + + try: + await test_func(client) + except Exception as e: + console.print(f"[red]Test '{name}' failed with error: {e}[/]") + console.print_exception(show_locals=False) + + console.rule("[bold green]All Tests Complete![/]", style="green") + + # Summary + console.print("\n[bold cyan]Configuration Hierarchy Summary:[/]") + console.print("1. [yellow]Request parameters[/] - Highest priority (temperature, base_url in API call)") + console.print("2. [yellow]Provider-specific env[/] - e.g., OPENAI_TEMPERATURE, GROQ_BASE_URL") + console.print("3. [yellow]Global env variables[/] - LLM_TEMPERATURE, LLM_BASE_URL") + console.print("4. [yellow]System defaults[/] - Lowest priority (provider/litellm defaults)") + console.print() + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + console.print("\n[yellow]Tests interrupted by user.[/]") + except Exception as e: + console.print(f"\n[bold red]An error occurred:[/]") + console.print_exception(show_locals=False) \ No newline at end of file From 2ad3fb5fc89e71cdb6090e971caeb5dd59690ecb Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Tue, 26 Aug 2025 23:18:35 +0530 Subject: [PATCH 016/119] feat(docker): improve docker error handling - Return comprehensive error messages along with status codes for api internal errors. - Fix fit_html property serialization issue in both /crawl and /crawl/stream endpoints - Add sanitization to ensure fit_html is always JSON-serializable (string or None) - Add comprehensive error handling test suite. --- deploy/docker/api.py | 6 + deploy/docker/server.py | 124 +++++++++++----- tests/docker/test_server_requests.py | 204 ++++++++++++++++++++++++++- 3 files changed, 296 insertions(+), 38 deletions(-) diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 53359e1f..850f5fd6 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -413,6 +413,9 @@ async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator) server_memory_mb = _get_memory_mb() result_dict = result.model_dump() result_dict['server_memory_mb'] = server_memory_mb + # Ensure fit_html is JSON-serializable + if "fit_html" in result_dict and not (result_dict["fit_html"] is None or isinstance(result_dict["fit_html"], str)): + result_dict["fit_html"] = None # If PDF exists, encode it to base64 if result_dict.get('pdf') is not None: result_dict['pdf'] = b64encode(result_dict['pdf']).decode('utf-8') @@ -493,6 +496,9 @@ async def handle_crawl_request( processed_results = [] for result in results: result_dict = result.model_dump() + # if fit_html is not a string, set it to None to avoid serialization errors + if "fit_html" in result_dict and not (result_dict["fit_html"] is None or isinstance(result_dict["fit_html"], str)): + result_dict["fit_html"] = None # If PDF exists, encode it to base64 if result_dict.get('pdf') is not None: result_dict['pdf'] = b64encode(result_dict['pdf']).decode('utf-8') diff --git a/deploy/docker/server.py b/deploy/docker/server.py index e453758a..b79324da 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -267,12 +267,26 @@ async def generate_html( Use when you need sanitized HTML structures for building schemas or further processing. """ cfg = CrawlerRunConfig() - async with AsyncWebCrawler(config=BrowserConfig()) as crawler: - results = await crawler.arun(url=body.url, config=cfg) - raw_html = results[0].html - from crawl4ai.utils import preprocess_html_for_schema - processed_html = preprocess_html_for_schema(raw_html) - return JSONResponse({"html": processed_html, "url": body.url, "success": True}) + try: + async with AsyncWebCrawler(config=BrowserConfig()) as crawler: + results = await crawler.arun(url=body.url, config=cfg) + # Check if the crawl was successful + if not results[0].success: + raise HTTPException( + status_code=500, + detail=results[0].error_message or "Crawl failed" + ) + + raw_html = results[0].html + from crawl4ai.utils import preprocess_html_for_schema + processed_html = preprocess_html_for_schema(raw_html) + return JSONResponse({"html": processed_html, "url": body.url, "success": True}) + except Exception as e: + # Log and raise as HTTP 500 for other exceptions + raise HTTPException( + status_code=500, + detail=str(e) + ) # Screenshot endpoint @@ -290,18 +304,29 @@ async def generate_screenshot( Use when you need an image snapshot of the rendered page. Its recommened to provide an output path to save the screenshot. Then in result instead of the screenshot you will get a path to the saved file. """ - cfg = CrawlerRunConfig( - screenshot=True, screenshot_wait_for=body.screenshot_wait_for) - async with AsyncWebCrawler(config=BrowserConfig()) as crawler: - results = await crawler.arun(url=body.url, config=cfg) - screenshot_data = results[0].screenshot - if body.output_path: - abs_path = os.path.abspath(body.output_path) - os.makedirs(os.path.dirname(abs_path), exist_ok=True) - with open(abs_path, "wb") as f: - f.write(base64.b64decode(screenshot_data)) - return {"success": True, "path": abs_path} - return {"success": True, "screenshot": screenshot_data} + try: + cfg = CrawlerRunConfig( + screenshot=True, screenshot_wait_for=body.screenshot_wait_for) + async with AsyncWebCrawler(config=BrowserConfig()) as crawler: + results = await crawler.arun(url=body.url, config=cfg) + if not results[0].success: + raise HTTPException( + status_code=500, + detail=results[0].error_message or "Crawl failed" + ) + screenshot_data = results[0].screenshot + if body.output_path: + abs_path = os.path.abspath(body.output_path) + os.makedirs(os.path.dirname(abs_path), exist_ok=True) + with open(abs_path, "wb") as f: + f.write(base64.b64decode(screenshot_data)) + return {"success": True, "path": abs_path} + return {"success": True, "screenshot": screenshot_data} + except Exception as e: + raise HTTPException( + status_code=500, + detail=str(e) + ) # PDF endpoint @@ -319,17 +344,28 @@ async def generate_pdf( Use when you need a printable or archivable snapshot of the page. It is recommended to provide an output path to save the PDF. Then in result instead of the PDF you will get a path to the saved file. """ - cfg = CrawlerRunConfig(pdf=True) - async with AsyncWebCrawler(config=BrowserConfig()) as crawler: - results = await crawler.arun(url=body.url, config=cfg) - pdf_data = results[0].pdf - if body.output_path: - abs_path = os.path.abspath(body.output_path) - os.makedirs(os.path.dirname(abs_path), exist_ok=True) - with open(abs_path, "wb") as f: - f.write(pdf_data) - return {"success": True, "path": abs_path} - return {"success": True, "pdf": base64.b64encode(pdf_data).decode()} + try: + cfg = CrawlerRunConfig(pdf=True) + async with AsyncWebCrawler(config=BrowserConfig()) as crawler: + results = await crawler.arun(url=body.url, config=cfg) + if not results[0].success: + raise HTTPException( + status_code=500, + detail=results[0].error_message or "Crawl failed" + ) + pdf_data = results[0].pdf + if body.output_path: + abs_path = os.path.abspath(body.output_path) + os.makedirs(os.path.dirname(abs_path), exist_ok=True) + with open(abs_path, "wb") as f: + f.write(pdf_data) + return {"success": True, "path": abs_path} + return {"success": True, "pdf": base64.b64encode(pdf_data).decode()} + except Exception as e: + raise HTTPException( + status_code=500, + detail=str(e) + ) @app.post("/execute_js") @@ -385,12 +421,23 @@ async def execute_js( ``` """ - cfg = CrawlerRunConfig(js_code=body.scripts) - async with AsyncWebCrawler(config=BrowserConfig()) as crawler: - results = await crawler.arun(url=body.url, config=cfg) - # Return JSON-serializable dict of the first CrawlResult - data = results[0].model_dump() - return JSONResponse(data) + try: + cfg = CrawlerRunConfig(js_code=body.scripts) + async with AsyncWebCrawler(config=BrowserConfig()) as crawler: + results = await crawler.arun(url=body.url, config=cfg) + if not results[0].success: + raise HTTPException( + status_code=500, + detail=results[0].error_message or "Crawl failed" + ) + # Return JSON-serializable dict of the first CrawlResult + data = results[0].model_dump() + return JSONResponse(data) + except Exception as e: + raise HTTPException( + status_code=500, + detail=str(e) + ) @app.get("/llm/{url:path}") @@ -438,13 +485,16 @@ async def crawl( """ if not crawl_request.urls: raise HTTPException(400, "At least one URL required") - res = await handle_crawl_request( + results = await handle_crawl_request( urls=crawl_request.urls, browser_config=crawl_request.browser_config, crawler_config=crawl_request.crawler_config, config=config, ) - return JSONResponse(res) + # check if all of the results are not successful + if all(not result["success"] for result in results["results"]): + raise HTTPException(500, f"Crawl request failed: {results['results'][0]['error_message']}") + return JSONResponse(results) @app.post("/crawl/stream") diff --git a/tests/docker/test_server_requests.py b/tests/docker/test_server_requests.py index 56d2ada4..034a2e86 100644 --- a/tests/docker/test_server_requests.py +++ b/tests/docker/test_server_requests.py @@ -635,7 +635,209 @@ class TestCrawlEndpoints: pytest.fail(f"LLM extracted content parsing or validation failed: {e}\nContent: {result['extracted_content']}") except Exception as e: # Catch any other unexpected error pytest.fail(f"An unexpected error occurred during LLM result processing: {e}\nContent: {result['extracted_content']}") - + + + # 7. Error Handling Tests + async def test_invalid_url_handling(self, async_client: httpx.AsyncClient): + """Test error handling for invalid URLs.""" + payload = { + "urls": ["invalid-url", "https://nonexistent-domain-12345.com"], + "browser_config": {"type": "BrowserConfig", "params": {"headless": True}}, + "crawler_config": {"type": "CrawlerRunConfig", "params": {"cache_mode": CacheMode.BYPASS.value}} + } + + response = await async_client.post("/crawl", json=payload) + # Should return 200 with failed results, not 500 + print(f"Status code: {response.status_code}") + print(f"Response: {response.text}") + assert response.status_code == 500 + data = response.json() + assert data["detail"].startswith("Crawl request failed:") + + async def test_mixed_success_failure_urls(self, async_client: httpx.AsyncClient): + """Test handling of mixed success/failure URLs.""" + payload = { + "urls": [ + SIMPLE_HTML_URL, # Should succeed + "https://nonexistent-domain-12345.com", # Should fail + "https://invalid-url-with-special-chars-!@#$%^&*()", # Should fail + ], + "browser_config": {"type": "BrowserConfig", "params": {"headless": True}}, + "crawler_config": { + "type": "CrawlerRunConfig", + "params": { + "cache_mode": CacheMode.BYPASS.value, + "markdown_generator": { + "type": "DefaultMarkdownGenerator", + "params": { + "content_filter": { + "type": "PruningContentFilter", + "params": {"threshold": 0.5} + } + } + } + } + } + } + + response = await async_client.post("/crawl", json=payload) + assert response.status_code == 200 + data = response.json() + assert data["success"] is True + assert len(data["results"]) == 3 + + success_count = 0 + failure_count = 0 + + for result in data["results"]: + if result["success"]: + success_count += 1 + else: + failure_count += 1 + assert "error_message" in result + assert len(result["error_message"]) > 0 + + assert success_count >= 1 # At least one should succeed + assert failure_count >= 1 # At least one should fail + + async def test_streaming_mixed_urls(self, async_client: httpx.AsyncClient): + """Test streaming with mixed success/failure URLs.""" + payload = { + "urls": [ + SIMPLE_HTML_URL, # Should succeed + "https://nonexistent-domain-12345.com", # Should fail + ], + "browser_config": {"type": "BrowserConfig", "params": {"headless": True}}, + "crawler_config": { + "type": "CrawlerRunConfig", + "params": { + "stream": True, + "cache_mode": CacheMode.BYPASS.value + } + } + } + + async with async_client.stream("POST", "/crawl/stream", json=payload) as response: + response.raise_for_status() + results = await process_streaming_response(response) + + assert len(results) == 2 + + success_count = 0 + failure_count = 0 + + for result in results: + if result["success"]: + success_count += 1 + assert result["url"] == SIMPLE_HTML_URL + else: + failure_count += 1 + assert "error_message" in result + assert result["error_message"] is not None + + assert success_count == 1 + assert failure_count == 1 + + async def test_markdown_endpoint_error_handling(self, async_client: httpx.AsyncClient): + """Test error handling for markdown endpoint.""" + # Test invalid URL + invalid_payload = {"url": "invalid-url", "f": "fit"} + response = await async_client.post("/md", json=invalid_payload) + # Should return 400 for invalid URL format + assert response.status_code == 400 + + # Test non-existent URL + nonexistent_payload = {"url": "https://nonexistent-domain-12345.com", "f": "fit"} + response = await async_client.post("/md", json=nonexistent_payload) + # Should return 500 for crawl failure + assert response.status_code == 500 + + async def test_html_endpoint_error_handling(self, async_client: httpx.AsyncClient): + """Test error handling for HTML endpoint.""" + # Test invalid URL + invalid_payload = {"url": "invalid-url"} + response = await async_client.post("/html", json=invalid_payload) + # Should return 500 for crawl failure + assert response.status_code == 500 + + async def test_screenshot_endpoint_error_handling(self, async_client: httpx.AsyncClient): + """Test error handling for screenshot endpoint.""" + # Test invalid URL + invalid_payload = {"url": "invalid-url"} + response = await async_client.post("/screenshot", json=invalid_payload) + # Should return 500 for crawl failure + assert response.status_code == 500 + + async def test_pdf_endpoint_error_handling(self, async_client: httpx.AsyncClient): + """Test error handling for PDF endpoint.""" + # Test invalid URL + invalid_payload = {"url": "invalid-url"} + response = await async_client.post("/pdf", json=invalid_payload) + # Should return 500 for crawl failure + assert response.status_code == 500 + + async def test_execute_js_endpoint_error_handling(self, async_client: httpx.AsyncClient): + """Test error handling for execute_js endpoint.""" + # Test invalid URL + invalid_payload = {"url": "invalid-url", "scripts": ["return document.title;"]} + response = await async_client.post("/execute_js", json=invalid_payload) + # Should return 500 for crawl failure + assert response.status_code == 500 + + async def test_llm_endpoint_error_handling(self, async_client: httpx.AsyncClient): + """Test error handling for LLM endpoint.""" + # Test missing query parameter + response = await async_client.get("/llm/https://example.com") + assert response.status_code == 422 # FastAPI validation error, not 400 + + # Test invalid URL + response = await async_client.get("/llm/invalid-url?q=test") + # Should return 500 for crawl failure + assert response.status_code == 500 + + async def test_ask_endpoint_error_handling(self, async_client: httpx.AsyncClient): + """Test error handling for ask endpoint.""" + # Test invalid context_type + response = await async_client.get("/ask?context_type=invalid") + assert response.status_code == 422 # Validation error + + # Test invalid score_ratio + response = await async_client.get("/ask?score_ratio=2.0") # > 1.0 + assert response.status_code == 422 # Validation error + + # Test invalid max_results + response = await async_client.get("/ask?max_results=0") # < 1 + assert response.status_code == 422 # Validation error + + async def test_config_dump_error_handling(self, async_client: httpx.AsyncClient): + """Test error handling for config dump endpoint.""" + # Test invalid code + invalid_payload = {"code": "invalid_code"} + response = await async_client.post("/config/dump", json=invalid_payload) + assert response.status_code == 400 + + # Test nested function calls (not allowed) + nested_payload = {"code": "CrawlerRunConfig(BrowserConfig())"} + response = await async_client.post("/config/dump", json=nested_payload) + assert response.status_code == 400 + + async def test_malformed_request_handling(self, async_client: httpx.AsyncClient): + """Test handling of malformed requests.""" + # Test missing required fields + malformed_payload = {"urls": []} # Missing browser_config and crawler_config + response = await async_client.post("/crawl", json=malformed_payload) + print(f"Response: {response.text}") + assert response.status_code == 422 # Validation error + + # Test empty URLs list + empty_urls_payload = { + "urls": [], + "browser_config": {"type": "BrowserConfig", "params": {}}, + "crawler_config": {"type": "CrawlerRunConfig", "params": {}} + } + response = await async_client.post("/crawl", json=empty_urls_payload) + assert response.status_code == 422 # "At least one URL required" + if __name__ == "__main__": # Define arguments for pytest programmatically # -v: verbose output From f7a3366f720a32ada460ec4dfbf8c418d7f7643f Mon Sep 17 00:00:00 2001 From: AHMET YILMAZ Date: Thu, 28 Aug 2025 17:21:49 +0800 Subject: [PATCH 017/119] #1375 : refactor(proxy) Deprecate 'proxy' parameter in BrowserConfig and enhance proxy string parsing - Updated ProxyConfig.from_string to support multiple proxy formats, including URLs with credentials. - Deprecated the 'proxy' parameter in BrowserConfig, replacing it with 'proxy_config' for better flexibility. - Added warnings for deprecated usage and clarified behavior when both parameters are provided. - Updated documentation and tests to reflect changes in proxy configuration handling. --- crawl4ai/async_configs.py | 61 ++++++++++++++------ crawl4ai/browser_manager.py | 20 ++++--- deploy/docker/c4ai-code-context.md | 19 +++--- docs/md_v2/advanced/proxy-security.md | 12 ++-- docs/md_v2/api/parameters.md | 2 +- test_scripts/test_proxy_deprecation.py | 70 +++++++++++++++++++++++ tests/async/test_0.4.2_browser_manager.py | 2 +- tests/memory/test_docker_config_gen.py | 2 +- tests/proxy/test_proxy_deprecation.py | 42 ++++++++++++++ 9 files changed, 188 insertions(+), 42 deletions(-) create mode 100644 test_scripts/test_proxy_deprecation.py create mode 100644 tests/proxy/test_proxy_deprecation.py diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 0c843b2b..6688a54b 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -1,5 +1,6 @@ import os from typing import Union +import warnings from .config import ( DEFAULT_PROVIDER, DEFAULT_PROVIDER_API_KEY, @@ -257,24 +258,39 @@ class ProxyConfig: @staticmethod def from_string(proxy_str: str) -> "ProxyConfig": - """Create a ProxyConfig from a string in the format 'ip:port:username:password'.""" - parts = proxy_str.split(":") - if len(parts) == 4: # ip:port:username:password + """Create a ProxyConfig from a string. + + Supported formats: + - 'http://username:password@ip:port' + - 'http://ip:port' + - 'socks5://ip:port' + - 'ip:port:username:password' + - 'ip:port' + """ + s = (proxy_str or "").strip() + # URL with credentials + if "@" in s and "://" in s: + auth_part, server_part = s.split("@", 1) + protocol, credentials = auth_part.split("://", 1) + if ":" in credentials: + username, password = credentials.split(":", 1) + return ProxyConfig( + server=f"{protocol}://{server_part}", + username=username, + password=password, + ) + # URL without credentials (keep scheme) + if "://" in s and "@" not in s: + return ProxyConfig(server=s) + # Colon separated forms + parts = s.split(":") + if len(parts) == 4: ip, port, username, password = parts - return ProxyConfig( - server=f"http://{ip}:{port}", - username=username, - password=password, - ip=ip - ) - elif len(parts) == 2: # ip:port only + return ProxyConfig(server=f"http://{ip}:{port}", username=username, password=password) + if len(parts) == 2: ip, port = parts - return ProxyConfig( - server=f"http://{ip}:{port}", - ip=ip - ) - else: - raise ValueError(f"Invalid proxy string format: {proxy_str}") + return ProxyConfig(server=f"http://{ip}:{port}") + raise ValueError(f"Invalid proxy string format: {proxy_str}") @staticmethod def from_dict(proxy_dict: Dict) -> "ProxyConfig": @@ -438,6 +454,7 @@ class BrowserConfig: host: str = "localhost", enable_stealth: bool = False, ): + self.browser_type = browser_type self.headless = headless self.browser_mode = browser_mode @@ -450,13 +467,23 @@ class BrowserConfig: if self.browser_type in ["firefox", "webkit"]: self.channel = "" self.chrome_channel = "" + if proxy: + warnings.warn("The 'proxy' parameter is deprecated and will be removed in a future release. Use 'proxy_config' instead.", DeprecationWarning) self.proxy = proxy self.proxy_config = proxy_config if isinstance(self.proxy_config, dict): self.proxy_config = ProxyConfig.from_dict(self.proxy_config) if isinstance(self.proxy_config, str): self.proxy_config = ProxyConfig.from_string(self.proxy_config) - + + if self.proxy and self.proxy_config: + warnings.warn("Both 'proxy' and 'proxy_config' are provided. 'proxy_config' will take precedence.", UserWarning) + print(f"[DEBUG] Both proxy and proxy_config provided. Setting proxy to None.") + self.proxy = None + elif self.proxy: + # Convert proxy string to ProxyConfig if proxy_config is not provided + self.proxy_config = ProxyConfig.from_string(self.proxy) + self.proxy = None self.viewport_width = viewport_width self.viewport_height = viewport_height diff --git a/crawl4ai/browser_manager.py b/crawl4ai/browser_manager.py index 8fed970c..48737fcb 100644 --- a/crawl4ai/browser_manager.py +++ b/crawl4ai/browser_manager.py @@ -15,6 +15,7 @@ from .js_snippet import load_js_script from .config import DOWNLOAD_PAGE_TIMEOUT from .async_configs import BrowserConfig, CrawlerRunConfig from .utils import get_chromium_path +import warnings BROWSER_DISABLE_OPTIONS = [ @@ -741,17 +742,18 @@ class BrowserManager: ) os.makedirs(browser_args["downloads_path"], exist_ok=True) - if self.config.proxy or self.config.proxy_config: + if self.config.proxy: + warnings.warn( + "BrowserConfig.proxy is deprecated and ignored. Use proxy_config instead.", + DeprecationWarning, + ) + if self.config.proxy_config: from playwright.async_api import ProxySettings - proxy_settings = ( - ProxySettings(server=self.config.proxy) - if self.config.proxy - else ProxySettings( - server=self.config.proxy_config.server, - username=self.config.proxy_config.username, - password=self.config.proxy_config.password, - ) + proxy_settings = ProxySettings( + server=self.config.proxy_config.server, + username=self.config.proxy_config.username, + password=self.config.proxy_config.password, ) browser_args["proxy"] = proxy_settings diff --git a/deploy/docker/c4ai-code-context.md b/deploy/docker/c4ai-code-context.md index eb29b94c..c18fbc78 100644 --- a/deploy/docker/c4ai-code-context.md +++ b/deploy/docker/c4ai-code-context.md @@ -7520,17 +7520,18 @@ class BrowserManager: ) os.makedirs(browser_args["downloads_path"], exist_ok=True) - if self.config.proxy or self.config.proxy_config: + if self.config.proxy: + warnings.warn( + "BrowserConfig.proxy is deprecated and ignored. Use proxy_config instead.", + DeprecationWarning, + ) + if self.config.proxy_config: from playwright.async_api import ProxySettings - proxy_settings = ( - ProxySettings(server=self.config.proxy) - if self.config.proxy - else ProxySettings( - server=self.config.proxy_config.server, - username=self.config.proxy_config.username, - password=self.config.proxy_config.password, - ) + proxy_settings = ProxySettings( + server=self.config.proxy_config.server, + username=self.config.proxy_config.username, + password=self.config.proxy_config.password, ) browser_args["proxy"] = proxy_settings diff --git a/docs/md_v2/advanced/proxy-security.md b/docs/md_v2/advanced/proxy-security.md index 13191cd7..d1c868b2 100644 --- a/docs/md_v2/advanced/proxy-security.md +++ b/docs/md_v2/advanced/proxy-security.md @@ -7,13 +7,13 @@ Simple proxy configuration with `BrowserConfig`: ```python from crawl4ai.async_configs import BrowserConfig -# Using proxy URL -browser_config = BrowserConfig(proxy="http://proxy.example.com:8080") +# Using HTTP proxy +browser_config = BrowserConfig(proxy_config={"server": "http://proxy.example.com:8080"}) async with AsyncWebCrawler(config=browser_config) as crawler: result = await crawler.arun(url="https://example.com") # Using SOCKS proxy -browser_config = BrowserConfig(proxy="socks5://proxy.example.com:1080") +browser_config = BrowserConfig(proxy_config={"server": "socks5://proxy.example.com:1080"}) async with AsyncWebCrawler(config=browser_config) as crawler: result = await crawler.arun(url="https://example.com") ``` @@ -25,7 +25,11 @@ Use an authenticated proxy with `BrowserConfig`: ```python from crawl4ai.async_configs import BrowserConfig -browser_config = BrowserConfig(proxy="http://[username]:[password]@[host]:[port]") +browser_config = BrowserConfig(proxy_config={ + "server": "http://[host]:[port]", + "username": "[username]", + "password": "[password]", +}) async with AsyncWebCrawler(config=browser_config) as crawler: result = await crawler.arun(url="https://example.com") ``` diff --git a/docs/md_v2/api/parameters.md b/docs/md_v2/api/parameters.md index ba526fb7..4418b5b5 100644 --- a/docs/md_v2/api/parameters.md +++ b/docs/md_v2/api/parameters.md @@ -23,7 +23,7 @@ browser_cfg = BrowserConfig( | **`headless`** | `bool` (default: `True`) | Headless means no visible UI. `False` is handy for debugging. | | **`viewport_width`** | `int` (default: `1080`) | Initial page width (in px). Useful for testing responsive layouts. | | **`viewport_height`** | `int` (default: `600`) | Initial page height (in px). | -| **`proxy`** | `str` (default: `None`) | Single-proxy URL if you want all traffic to go through it, e.g. `"http://user:pass@proxy:8080"`. | +| **`proxy`** | `str` (deprecated) | Deprecated. Use `proxy_config` instead. If set, it will be auto-converted internally. | | **`proxy_config`** | `dict` (default: `None`) | For advanced or multi-proxy needs, specify details like `{"server": "...", "username": "...", ...}`. | | **`use_persistent_context`** | `bool` (default: `False`) | If `True`, uses a **persistent** browser context (keep cookies, sessions across runs). Also sets `use_managed_browser=True`. | | **`user_data_dir`** | `str or None` (default: `None`) | Directory to store user data (profiles, cookies). Must be set if you want permanent sessions. | diff --git a/test_scripts/test_proxy_deprecation.py b/test_scripts/test_proxy_deprecation.py new file mode 100644 index 00000000..daf8d8bc --- /dev/null +++ b/test_scripts/test_proxy_deprecation.py @@ -0,0 +1,70 @@ +import sys +import warnings + +from crawl4ai.async_configs import BrowserConfig, ProxyConfig + + +def main() -> int: + warnings.simplefilter("always", DeprecationWarning) + + # Case 1: Using deprecated proxy string should emit DeprecationWarning and auto-convert + captured = [] + proxy_str = "23.95.150.145:6114:username:password" + with warnings.catch_warnings(record=True) as w: + cfg = BrowserConfig(proxy=proxy_str, headless=True) + captured = [m for m in w if issubclass(m.category, DeprecationWarning)] + + if not captured: + print("[FAIL] No DeprecationWarning emitted for BrowserConfig(proxy=...) usage.") + return 1 + + if cfg.proxy is not None: + print("[FAIL] cfg.proxy should be None after auto-conversion.") + return 1 + + if not isinstance(cfg.proxy_config, ProxyConfig): + print("[FAIL] cfg.proxy_config should be a ProxyConfig instance after auto-conversion.") + return 1 + + # Basic sanity checks on auto-parsed proxy_config + if not cfg.proxy_config.server or ":" not in (cfg.proxy_config.server or ""): + print("[FAIL] proxy_config.server appears invalid after conversion:", cfg.proxy_config.server) + return 1 + + if not cfg.proxy_config.username or not cfg.proxy_config.password: + print("[FAIL] proxy_config credentials missing after conversion.") + return 1 + + print("[OK] DeprecationWarning captured and proxy auto-converted to proxy_config.") + + # Case 2: Using proxy_config directly should not emit DeprecationWarning + with warnings.catch_warnings(record=True) as w2: + cfg2 = BrowserConfig( + proxy_config={ + "server": "http://127.0.0.1:8080", + "username": "u", + "password": "p", + }, + headless=True, + ) + + if any(issubclass(m.category, DeprecationWarning) for m in w2): + print("[FAIL] Unexpected DeprecationWarning when using proxy_config.") + return 1 + + if cfg2.proxy is not None: + print("[FAIL] cfg2.proxy should be None (only proxy_config should be used).") + return 1 + + if not isinstance(cfg2.proxy_config, ProxyConfig): + print("[FAIL] cfg2.proxy_config should be a ProxyConfig instance.") + return 1 + + print("[OK] proxy_config path works without deprecation warnings.") + print("All checks passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/tests/async/test_0.4.2_browser_manager.py b/tests/async/test_0.4.2_browser_manager.py index 21b4be11..635b731f 100644 --- a/tests/async/test_0.4.2_browser_manager.py +++ b/tests/async/test_0.4.2_browser_manager.py @@ -112,7 +112,7 @@ async def test_proxy_settings(): headless=True, verbose=False, user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36", - proxy="http://127.0.0.1:8080", # Assuming local proxy server for test + proxy_config={"server": "http://127.0.0.1:8080"}, # Assuming local proxy server for test use_managed_browser=False, use_persistent_context=False, ) as crawler: diff --git a/tests/memory/test_docker_config_gen.py b/tests/memory/test_docker_config_gen.py index ae6e533c..41beb30a 100644 --- a/tests/memory/test_docker_config_gen.py +++ b/tests/memory/test_docker_config_gen.py @@ -24,7 +24,7 @@ CASES = [ # --- BrowserConfig variants --- "BrowserConfig()", "BrowserConfig(headless=False, extra_args=['--disable-gpu'])", - "BrowserConfig(browser_mode='builtin', proxy='http://1.2.3.4:8080')", + "BrowserConfig(browser_mode='builtin', proxy_config={'server': 'http://1.2.3.4:8080'})", ] for code in CASES: diff --git a/tests/proxy/test_proxy_deprecation.py b/tests/proxy/test_proxy_deprecation.py new file mode 100644 index 00000000..95ccfc70 --- /dev/null +++ b/tests/proxy/test_proxy_deprecation.py @@ -0,0 +1,42 @@ +import warnings + +import pytest + +from crawl4ai.async_configs import BrowserConfig, ProxyConfig + + +def test_browser_config_proxy_string_emits_deprecation_and_autoconverts(): + warnings.simplefilter("always", DeprecationWarning) + + proxy_str = "23.95.150.145:6114:username:password" + with warnings.catch_warnings(record=True) as caught: + cfg = BrowserConfig(proxy=proxy_str, headless=True) + + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert dep_warnings, "Expected DeprecationWarning when using BrowserConfig(proxy=...)" + + assert cfg.proxy is None, "cfg.proxy should be None after auto-conversion" + assert isinstance(cfg.proxy_config, ProxyConfig), "cfg.proxy_config should be ProxyConfig instance" + assert cfg.proxy_config.username == "username" + assert cfg.proxy_config.password == "password" + assert cfg.proxy_config.server.startswith("http://") + assert cfg.proxy_config.server.endswith(":6114") + + +def test_browser_config_with_proxy_config_emits_no_deprecation(): + warnings.simplefilter("always", DeprecationWarning) + + with warnings.catch_warnings(record=True) as caught: + cfg = BrowserConfig( + headless=True, + proxy_config={ + "server": "http://127.0.0.1:8080", + "username": "u", + "password": "p", + }, + ) + + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert not dep_warnings, "Did not expect DeprecationWarning when using proxy_config" + assert cfg.proxy is None + assert isinstance(cfg.proxy_config, ProxyConfig) From 4ed33fce9eea96ba75b741a21922149724a620f1 Mon Sep 17 00:00:00 2001 From: AHMET YILMAZ Date: Thu, 28 Aug 2025 17:26:10 +0800 Subject: [PATCH 018/119] Remove deprecated test for 'proxy' parameter in BrowserConfig and update .gitignore to include test_scripts directory. --- .gitignore | 2 +- test_scripts/test_proxy_deprecation.py | 70 -------------------------- 2 files changed, 1 insertion(+), 71 deletions(-) delete mode 100644 test_scripts/test_proxy_deprecation.py diff --git a/.gitignore b/.gitignore index 6277b5cf..03c5b355 100644 --- a/.gitignore +++ b/.gitignore @@ -265,7 +265,7 @@ CLAUDE.md tests/**/test_site tests/**/reports tests/**/benchmark_reports - +test_scripts/ docs/**/data .codecat/ diff --git a/test_scripts/test_proxy_deprecation.py b/test_scripts/test_proxy_deprecation.py deleted file mode 100644 index daf8d8bc..00000000 --- a/test_scripts/test_proxy_deprecation.py +++ /dev/null @@ -1,70 +0,0 @@ -import sys -import warnings - -from crawl4ai.async_configs import BrowserConfig, ProxyConfig - - -def main() -> int: - warnings.simplefilter("always", DeprecationWarning) - - # Case 1: Using deprecated proxy string should emit DeprecationWarning and auto-convert - captured = [] - proxy_str = "23.95.150.145:6114:username:password" - with warnings.catch_warnings(record=True) as w: - cfg = BrowserConfig(proxy=proxy_str, headless=True) - captured = [m for m in w if issubclass(m.category, DeprecationWarning)] - - if not captured: - print("[FAIL] No DeprecationWarning emitted for BrowserConfig(proxy=...) usage.") - return 1 - - if cfg.proxy is not None: - print("[FAIL] cfg.proxy should be None after auto-conversion.") - return 1 - - if not isinstance(cfg.proxy_config, ProxyConfig): - print("[FAIL] cfg.proxy_config should be a ProxyConfig instance after auto-conversion.") - return 1 - - # Basic sanity checks on auto-parsed proxy_config - if not cfg.proxy_config.server or ":" not in (cfg.proxy_config.server or ""): - print("[FAIL] proxy_config.server appears invalid after conversion:", cfg.proxy_config.server) - return 1 - - if not cfg.proxy_config.username or not cfg.proxy_config.password: - print("[FAIL] proxy_config credentials missing after conversion.") - return 1 - - print("[OK] DeprecationWarning captured and proxy auto-converted to proxy_config.") - - # Case 2: Using proxy_config directly should not emit DeprecationWarning - with warnings.catch_warnings(record=True) as w2: - cfg2 = BrowserConfig( - proxy_config={ - "server": "http://127.0.0.1:8080", - "username": "u", - "password": "p", - }, - headless=True, - ) - - if any(issubclass(m.category, DeprecationWarning) for m in w2): - print("[FAIL] Unexpected DeprecationWarning when using proxy_config.") - return 1 - - if cfg2.proxy is not None: - print("[FAIL] cfg2.proxy should be None (only proxy_config should be used).") - return 1 - - if not isinstance(cfg2.proxy_config, ProxyConfig): - print("[FAIL] cfg2.proxy_config should be a ProxyConfig instance.") - return 1 - - print("[OK] proxy_config path works without deprecation warnings.") - print("All checks passed.") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) - From f566c5a376a452442c2905dc7f83dfa654117fc1 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Thu, 28 Aug 2025 17:38:40 +0800 Subject: [PATCH 019/119] feat: add preserve_https_for_internal_links flag to maintain HTTPS during crawling. Ref #1410 Added a new `preserve_https_for_internal_links` configuration flag that preserves the original HTTPS scheme for same-domain links even when the server redirects to HTTP. --- crawl4ai/async_configs.py | 4 + crawl4ai/async_webcrawler.py | 2 + crawl4ai/content_scraping_strategy.py | 6 +- crawl4ai/utils.py | 41 +++- .../test_preserve_https_for_internal_links.py | 175 ++++++++++++++++++ 5 files changed, 224 insertions(+), 4 deletions(-) create mode 100644 tests/test_preserve_https_for_internal_links.py diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index a43b50a4..3476fd9e 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -1121,6 +1121,7 @@ class CrawlerRunConfig(): exclude_domains: list = None, exclude_internal_links: bool = False, score_links: bool = False, + preserve_https_for_internal_links: bool = False, # Debugging and Logging Parameters verbose: bool = True, log_console: bool = False, @@ -1244,6 +1245,7 @@ class CrawlerRunConfig(): self.exclude_domains = exclude_domains or [] self.exclude_internal_links = exclude_internal_links self.score_links = score_links + self.preserve_https_for_internal_links = preserve_https_for_internal_links # Debugging and Logging Parameters self.verbose = verbose @@ -1517,6 +1519,7 @@ class CrawlerRunConfig(): exclude_domains=kwargs.get("exclude_domains", []), exclude_internal_links=kwargs.get("exclude_internal_links", False), score_links=kwargs.get("score_links", False), + preserve_https_for_internal_links=kwargs.get("preserve_https_for_internal_links", False), # Debugging and Logging Parameters verbose=kwargs.get("verbose", True), log_console=kwargs.get("log_console", False), @@ -1623,6 +1626,7 @@ class CrawlerRunConfig(): "exclude_domains": self.exclude_domains, "exclude_internal_links": self.exclude_internal_links, "score_links": self.score_links, + "preserve_https_for_internal_links": self.preserve_https_for_internal_links, "verbose": self.verbose, "log_console": self.log_console, "capture_network_requests": self.capture_network_requests, diff --git a/crawl4ai/async_webcrawler.py b/crawl4ai/async_webcrawler.py index ebd2859d..f12fc488 100644 --- a/crawl4ai/async_webcrawler.py +++ b/crawl4ai/async_webcrawler.py @@ -354,6 +354,7 @@ class AsyncWebCrawler: ############################################################### # Process the HTML content, Call CrawlerStrategy.process_html # ############################################################### + from urllib.parse import urlparse crawl_result: CrawlResult = await self.aprocess_html( url=url, html=html, @@ -364,6 +365,7 @@ class AsyncWebCrawler: verbose=config.verbose, is_raw_html=True if url.startswith("raw:") else False, redirected_url=async_response.redirected_url, + original_scheme=urlparse(url).scheme, **kwargs, ) diff --git a/crawl4ai/content_scraping_strategy.py b/crawl4ai/content_scraping_strategy.py index 9ef0e616..d9095e49 100644 --- a/crawl4ai/content_scraping_strategy.py +++ b/crawl4ai/content_scraping_strategy.py @@ -258,7 +258,11 @@ class LXMLWebScrapingStrategy(ContentScrapingStrategy): continue try: - normalized_href = normalize_url(href, url) + normalized_href = normalize_url( + href, url, + preserve_https=kwargs.get('preserve_https_for_internal_links', False), + original_scheme=kwargs.get('original_scheme') + ) link_data = { "href": normalized_href, "text": link.text_content().strip(), diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py index 73f1d2a3..facc3781 100644 --- a/crawl4ai/utils.py +++ b/crawl4ai/utils.py @@ -2146,7 +2146,9 @@ def normalize_url( drop_query_tracking=True, sort_query=True, keep_fragment=False, - extra_drop_params=None + extra_drop_params=None, + preserve_https=False, + original_scheme=None ): """ Extended URL normalizer @@ -2176,6 +2178,17 @@ def normalize_url( # Resolve relative paths first full_url = urljoin(base_url, href.strip()) + + # Preserve HTTPS if requested and original scheme was HTTPS + if preserve_https and original_scheme == 'https': + parsed_full = urlparse(full_url) + parsed_base = urlparse(base_url) + # Only preserve HTTPS for same-domain links (not protocol-relative URLs) + # Protocol-relative URLs (//example.com) should follow the base URL's scheme + if (parsed_full.scheme == 'http' and + parsed_full.netloc == parsed_base.netloc and + not href.strip().startswith('//')): + full_url = full_url.replace('http://', 'https://', 1) # Parse once, edit parts, then rebuild parsed = urlparse(full_url) @@ -2225,7 +2238,7 @@ def normalize_url( return normalized -def normalize_url_for_deep_crawl(href, base_url): +def normalize_url_for_deep_crawl(href, base_url, preserve_https=False, original_scheme=None): """Normalize URLs to ensure consistent format""" from urllib.parse import urljoin, urlparse, urlunparse, parse_qs, urlencode @@ -2236,6 +2249,17 @@ def normalize_url_for_deep_crawl(href, base_url): # Use urljoin to handle relative URLs full_url = urljoin(base_url, href.strip()) + # Preserve HTTPS if requested and original scheme was HTTPS + if preserve_https and original_scheme == 'https': + parsed_full = urlparse(full_url) + parsed_base = urlparse(base_url) + # Only preserve HTTPS for same-domain links (not protocol-relative URLs) + # Protocol-relative URLs (//example.com) should follow the base URL's scheme + if (parsed_full.scheme == 'http' and + parsed_full.netloc == parsed_base.netloc and + not href.strip().startswith('//')): + full_url = full_url.replace('http://', 'https://', 1) + # Parse the URL for normalization parsed = urlparse(full_url) @@ -2273,7 +2297,7 @@ def normalize_url_for_deep_crawl(href, base_url): return normalized @lru_cache(maxsize=10000) -def efficient_normalize_url_for_deep_crawl(href, base_url): +def efficient_normalize_url_for_deep_crawl(href, base_url, preserve_https=False, original_scheme=None): """Efficient URL normalization with proper parsing""" from urllib.parse import urljoin @@ -2283,6 +2307,17 @@ def efficient_normalize_url_for_deep_crawl(href, base_url): # Resolve relative URLs full_url = urljoin(base_url, href.strip()) + # Preserve HTTPS if requested and original scheme was HTTPS + if preserve_https and original_scheme == 'https': + parsed_full = urlparse(full_url) + parsed_base = urlparse(base_url) + # Only preserve HTTPS for same-domain links (not protocol-relative URLs) + # Protocol-relative URLs (//example.com) should follow the base URL's scheme + if (parsed_full.scheme == 'http' and + parsed_full.netloc == parsed_base.netloc and + not href.strip().startswith('//')): + full_url = full_url.replace('http://', 'https://', 1) + # Use proper URL parsing parsed = urlparse(full_url) diff --git a/tests/test_preserve_https_for_internal_links.py b/tests/test_preserve_https_for_internal_links.py new file mode 100644 index 00000000..8988f1c9 --- /dev/null +++ b/tests/test_preserve_https_for_internal_links.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +""" +Final test and demo for HTTPS preservation feature (Issue #1410) + +This demonstrates how the preserve_https_for_internal_links flag +prevents HTTPS downgrade when servers redirect to HTTP. +""" + +import sys +import os +from urllib.parse import urljoin, urlparse + +def demonstrate_issue(): + """Show the problem: HTTPS -> HTTP redirect causes HTTP links""" + + print("=" * 60) + print("DEMONSTRATING THE ISSUE") + print("=" * 60) + + # Simulate what happens during crawling + original_url = "https://quotes.toscrape.com/tag/deep-thoughts" + redirected_url = "http://quotes.toscrape.com/tag/deep-thoughts/" # Server redirects to HTTP + + # Extract a relative link + relative_link = "/author/Albert-Einstein" + + # Standard URL joining uses the redirected (HTTP) base + resolved_url = urljoin(redirected_url, relative_link) + + print(f"Original URL: {original_url}") + print(f"Redirected to: {redirected_url}") + print(f"Relative link: {relative_link}") + print(f"Resolved link: {resolved_url}") + print(f"\n❌ Problem: Link is now HTTP instead of HTTPS!") + + return resolved_url + +def demonstrate_solution(): + """Show the solution: preserve HTTPS for internal links""" + + print("\n" + "=" * 60) + print("DEMONSTRATING THE SOLUTION") + print("=" * 60) + + # Our normalize_url with HTTPS preservation + def normalize_url_with_preservation(href, base_url, preserve_https=False, original_scheme=None): + """Normalize URL with optional HTTPS preservation""" + + # Standard resolution + full_url = urljoin(base_url, href.strip()) + + # Preserve HTTPS if requested + if preserve_https and original_scheme == 'https': + parsed_full = urlparse(full_url) + parsed_base = urlparse(base_url) + + # Only for same-domain links + if parsed_full.scheme == 'http' and parsed_full.netloc == parsed_base.netloc: + full_url = full_url.replace('http://', 'https://', 1) + print(f" → Preserved HTTPS for {parsed_full.netloc}") + + return full_url + + # Same scenario as before + original_url = "https://quotes.toscrape.com/tag/deep-thoughts" + redirected_url = "http://quotes.toscrape.com/tag/deep-thoughts/" + relative_link = "/author/Albert-Einstein" + + # Without preservation (current behavior) + resolved_without = normalize_url_with_preservation( + relative_link, redirected_url, + preserve_https=False, original_scheme='https' + ) + + print(f"\nWithout preservation:") + print(f" Result: {resolved_without}") + + # With preservation (new feature) + resolved_with = normalize_url_with_preservation( + relative_link, redirected_url, + preserve_https=True, original_scheme='https' + ) + + print(f"\nWith preservation (preserve_https_for_internal_links=True):") + print(f" Result: {resolved_with}") + print(f"\n✅ Solution: Internal link stays HTTPS!") + + return resolved_with + +def test_edge_cases(): + """Test important edge cases""" + + print("\n" + "=" * 60) + print("EDGE CASES") + print("=" * 60) + + from urllib.parse import urljoin, urlparse + + def preserve_https(href, base_url, original_scheme): + """Helper to test preservation logic""" + full_url = urljoin(base_url, href) + + if original_scheme == 'https': + parsed_full = urlparse(full_url) + parsed_base = urlparse(base_url) + # Fixed: check for protocol-relative URLs + if (parsed_full.scheme == 'http' and + parsed_full.netloc == parsed_base.netloc and + not href.strip().startswith('//')): + full_url = full_url.replace('http://', 'https://', 1) + + return full_url + + test_cases = [ + # (description, href, base_url, original_scheme, should_be_https) + ("External link", "http://other.com/page", "http://example.com", "https", False), + ("Already HTTPS", "/page", "https://example.com", "https", True), + ("No original HTTPS", "/page", "http://example.com", "http", False), + ("Subdomain", "/page", "http://sub.example.com", "https", True), + ("Protocol-relative", "//example.com/page", "http://example.com", "https", False), + ] + + for desc, href, base_url, orig_scheme, should_be_https in test_cases: + result = preserve_https(href, base_url, orig_scheme) + is_https = result.startswith('https://') + status = "✅" if is_https == should_be_https else "❌" + + print(f"\n{status} {desc}:") + print(f" Input: {href} + {base_url}") + print(f" Result: {result}") + print(f" Expected HTTPS: {should_be_https}, Got: {is_https}") + +def usage_example(): + """Show how to use the feature in crawl4ai""" + + print("\n" + "=" * 60) + print("USAGE IN CRAWL4AI") + print("=" * 60) + + print(""" +To enable HTTPS preservation in your crawl4ai code: + +```python +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig + +async with AsyncWebCrawler() as crawler: + config = CrawlerRunConfig( + preserve_https_for_internal_links=True # Enable HTTPS preservation + ) + + result = await crawler.arun( + url="https://example.com", + config=config + ) + + # All internal links will maintain HTTPS even if + # the server redirects to HTTP +``` + +This is especially useful for: +- Sites that redirect HTTPS to HTTP but still support HTTPS +- Security-conscious crawling where you want to stay on HTTPS +- Avoiding mixed content issues in downstream processing +""") + +if __name__ == "__main__": + # Run all demonstrations + demonstrate_issue() + demonstrate_solution() + test_edge_cases() + usage_example() + + print("\n" + "=" * 60) + print("✅ All tests complete!") + print("=" * 60) \ No newline at end of file From bdacf61ca91d046a9bfebe154c1f561055c303ef Mon Sep 17 00:00:00 2001 From: ntohidi Date: Thu, 28 Aug 2025 17:48:12 +0800 Subject: [PATCH 020/119] feat: update documentation for preserve_https_for_internal_links. ref #1410 --- CHANGELOG.md | 10 ++++++++++ docs/md_v2/api/parameters.md | 1 + docs/md_v2/core/deep-crawling.md | 11 +++++++++++ 3 files changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9788caf2..ce63516f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to Crawl4AI will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added +- **🔒 HTTPS Preservation for Internal Links**: New `preserve_https_for_internal_links` configuration flag + - Maintains HTTPS scheme for internal links even when servers redirect to HTTP + - Prevents security downgrades during deep crawling + - Useful for security-conscious crawling and sites supporting both protocols + - Fully backward compatible with opt-in flag (default: `False`) + - Fixes issue #1410 where HTTPS URLs were being downgraded to HTTP + ## [0.7.3] - 2025-08-09 ### Added diff --git a/docs/md_v2/api/parameters.md b/docs/md_v2/api/parameters.md index ba526fb7..47f719c8 100644 --- a/docs/md_v2/api/parameters.md +++ b/docs/md_v2/api/parameters.md @@ -155,6 +155,7 @@ If your page is a single-page app with repeated JS updates, set `js_only=True` i | **`exclude_external_links`** | `bool` (False) | Removes all links pointing outside the current domain. | | **`exclude_social_media_links`** | `bool` (False) | Strips links specifically to social sites (like Facebook or Twitter). | | **`exclude_domains`** | `list` ([]) | Provide a custom list of domains to exclude (like `["ads.com", "trackers.io"]`). | +| **`preserve_https_for_internal_links`** | `bool` (False) | If `True`, preserves HTTPS scheme for internal links even when the server redirects to HTTP. Useful for security-conscious crawling. | Use these for link-level content filtering (often to keep crawls “internal” or to remove spammy domains). diff --git a/docs/md_v2/core/deep-crawling.md b/docs/md_v2/core/deep-crawling.md index 00834787..93760f23 100644 --- a/docs/md_v2/core/deep-crawling.md +++ b/docs/md_v2/core/deep-crawling.md @@ -472,6 +472,17 @@ Note that for BestFirstCrawlingStrategy, score_threshold is not needed since pag 5.**Balance breadth vs. depth.** Choose your strategy wisely - BFS for comprehensive coverage, DFS for deep exploration, BestFirst for focused relevance-based crawling. +6.**Preserve HTTPS for security.** If crawling HTTPS sites that redirect to HTTP, use `preserve_https_for_internal_links=True` to maintain secure connections: + +```python +config = CrawlerRunConfig( + deep_crawl_strategy=BFSDeepCrawlStrategy(max_depth=2), + preserve_https_for_internal_links=True # Keep HTTPS even if server redirects to HTTP +) +``` + +This is especially useful for security-conscious crawling or when dealing with sites that support both protocols. + --- ## 10. Summary & Next Steps From 70f473b84de1faec347e468067394bac4e7467ae Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Thu, 28 Aug 2025 19:27:33 +0530 Subject: [PATCH 021/119] fix: drop Python 3.9 support and require Python >=3.10. The library no longer supports Python 3.9 and so it was important to drop all references to python 3.9. Following changes have been made: - pyproject.toml: set requires-python to ">=3.10"; remove 3.9 classifier - setup.py: set python_requires to ">=3.10"; remove 3.9 classifier - docs: update Python version mentions - deploy/docker/c4ai-doc-context.md: options -> 3.10, 3.11, 3.12, 3.13 --- deploy/docker/c4ai-doc-context.md | 2 +- pyproject.toml | 3 +-- setup.py | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/deploy/docker/c4ai-doc-context.md b/deploy/docker/c4ai-doc-context.md index 74ad794f..abfd3637 100644 --- a/deploy/docker/c4ai-doc-context.md +++ b/deploy/docker/c4ai-doc-context.md @@ -2241,7 +2241,7 @@ docker build -t crawl4ai | Argument | Description | Default | Options | |----------|-------------|---------|----------| -| PYTHON_VERSION | Python version | 3.10 | 3.8, 3.9, 3.10 | +| PYTHON_VERSION | Python version | 3.10 | 3.10, 3.11, 3.12, 3.13 | | INSTALL_TYPE | Feature set | default | default, all, torch, transformer | | ENABLE_GPU | GPU support | false | true, false | | APP_HOME | Install path | /app | any valid path | diff --git a/pyproject.toml b/pyproject.toml index 9b00bd28..b2853152 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "Crawl4AI" dynamic = ["version"] description = "🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & scraper" readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.10" license = "Apache-2.0" authors = [ {name = "Unclecode", email = "unclecode@kidocode.com"} @@ -51,7 +51,6 @@ classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", diff --git a/setup.py b/setup.py index a0b91041..4d6e0575 100644 --- a/setup.py +++ b/setup.py @@ -56,11 +56,10 @@ setup( "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", ], - python_requires=">=3.9", + python_requires=">=3.10", ) From 9749e2832d3b92d97381004c9743e23198f4fe92 Mon Sep 17 00:00:00 2001 From: nafeqq-1306 Date: Fri, 29 Aug 2025 10:20:47 +0800 Subject: [PATCH 022/119] issue #1329 refactor(crawler): move unwanted properties to CrawlerRunConfig class --- .yoyo/snapshot | 1 + crawl4ai/async_configs.py | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) create mode 160000 .yoyo/snapshot diff --git a/.yoyo/snapshot b/.yoyo/snapshot new file mode 160000 index 00000000..d2698b0e --- /dev/null +++ b/.yoyo/snapshot @@ -0,0 +1 @@ +Subproject commit d2698b0e1a35fea0eff87d7e2c9256125d56ad21 diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 174ec3e4..37f5e8cf 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -813,12 +813,6 @@ class HTTPCrawlerConfig: return HTTPCrawlerConfig.from_kwargs(config) class CrawlerRunConfig(): - _UNWANTED_PROPS = { - 'disable_cache' : 'Instead, use cache_mode=CacheMode.DISABLED', - 'bypass_cache' : 'Instead, use cache_mode=CacheMode.BYPASS', - 'no_cache_read' : 'Instead, use cache_mode=CacheMode.WRITE_ONLY', - 'no_cache_write' : 'Instead, use cache_mode=CacheMode.READ_ONLY', - } """ Configuration class for controlling how the crawler runs each crawl operation. @@ -1023,6 +1017,12 @@ class CrawlerRunConfig(): url: str = None # This is not a compulsory parameter """ + _UNWANTED_PROPS = { + 'disable_cache' : 'Instead, use cache_mode=CacheMode.DISABLED', + 'bypass_cache' : 'Instead, use cache_mode=CacheMode.BYPASS', + 'no_cache_read' : 'Instead, use cache_mode=CacheMode.WRITE_ONLY', + 'no_cache_write' : 'Instead, use cache_mode=CacheMode.READ_ONLY', + } def __init__( self, From 6e728096fa79f347a94a7be65a52bda47b9ba785 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 1 Sep 2025 12:48:16 +0800 Subject: [PATCH 023/119] fix(auth): fixed Docker JWT authentication. ref #1442 --- deploy/docker/auth.py | 34 ++++++++++++++++++++++++++-------- deploy/docker/config.yml | 4 ++-- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/deploy/docker/auth.py b/deploy/docker/auth.py index f9e75d78..6fcef339 100644 --- a/deploy/docker/auth.py +++ b/deploy/docker/auth.py @@ -28,25 +28,43 @@ def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) - signing_key = get_jwk_from_secret(SECRET_KEY) return instance.encode(to_encode, signing_key, alg='HS256') -def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)) -> Dict: +def verify_token(credentials: HTTPAuthorizationCredentials) -> Dict: """Verify the JWT token from the Authorization header.""" - - if credentials is None: - return None + + if not credentials or not credentials.credentials: + raise HTTPException( + status_code=401, + detail="No token provided", + headers={"WWW-Authenticate": "Bearer"} + ) + token = credentials.credentials verifying_key = get_jwk_from_secret(SECRET_KEY) try: payload = instance.decode(token, verifying_key, do_time_check=True, algorithms='HS256') return payload - except Exception: - raise HTTPException(status_code=401, detail="Invalid or expired token") + except Exception as e: + raise HTTPException( + status_code=401, + detail=f"Invalid or expired token: {str(e)}", + headers={"WWW-Authenticate": "Bearer"} + ) def get_token_dependency(config: Dict): """Return the token dependency if JWT is enabled, else a function that returns None.""" - + if config.get("security", {}).get("jwt_enabled", False): - return verify_token + def jwt_required(credentials: HTTPAuthorizationCredentials = Depends(security)) -> Dict: + """Enforce JWT authentication when enabled.""" + if credentials is None: + raise HTTPException( + status_code=401, + detail="Authentication required. Please provide a valid Bearer token.", + headers={"WWW-Authenticate": "Bearer"} + ) + return verify_token(credentials) + return jwt_required else: return lambda: None diff --git a/deploy/docker/config.yml b/deploy/docker/config.yml index f5046613..35371375 100644 --- a/deploy/docker/config.yml +++ b/deploy/docker/config.yml @@ -38,8 +38,8 @@ rate_limiting: # Security Configuration security: - enabled: false - jwt_enabled: false + enabled: false + jwt_enabled: false https_redirect: false trusted_hosts: ["*"] headers: From 6772134a3abdada83638e2bb2db3257259ca596f Mon Sep 17 00:00:00 2001 From: ntohidi Date: Tue, 2 Sep 2025 12:07:08 +0800 Subject: [PATCH 024/119] remove: delete unused yoyo snapshot subproject --- .yoyo/snapshot | 1 - 1 file changed, 1 deletion(-) delete mode 160000 .yoyo/snapshot diff --git a/.yoyo/snapshot b/.yoyo/snapshot deleted file mode 160000 index d2698b0e..00000000 --- a/.yoyo/snapshot +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d2698b0e1a35fea0eff87d7e2c9256125d56ad21 From 487839640f79c4b55cff4931b472ff9aea685ee1 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Tue, 2 Sep 2025 16:49:01 +0800 Subject: [PATCH 025/119] fix: raise error on last attempt failure in perform_completion_with_backoff. ref #989 --- crawl4ai/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py index a871bc91..046351e7 100644 --- a/crawl4ai/utils.py +++ b/crawl4ai/utils.py @@ -1790,6 +1790,10 @@ def perform_completion_with_backoff( except RateLimitError as e: print("Rate limit error:", str(e)) + if attempt == max_attempts - 1: + # Last attempt failed, raise the error. + raise + # Check if we have exhausted our max attempts if attempt < max_attempts - 1: # Calculate the delay and wait From 6a3b3e9d384bccc087912354d579713b68d4e339 Mon Sep 17 00:00:00 2001 From: AHMET YILMAZ Date: Wed, 3 Sep 2025 17:02:40 +0800 Subject: [PATCH 026/119] Commit without API --- deploy/docker/server.py | 11 +++++- deploy/docker/static/playground/index.html | 44 +++++++++++++++------- tests/docker/test_server_requests.py | 33 ++++++++++++++++ 3 files changed, 74 insertions(+), 14 deletions(-) diff --git a/deploy/docker/server.py b/deploy/docker/server.py index b79324da..57d78d06 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -482,9 +482,14 @@ async def crawl( ): """ Crawl a list of URLs and return the results as JSON. + For streaming responses, use /crawl/stream endpoint. """ if not crawl_request.urls: raise HTTPException(400, "At least one URL required") + # Check whether it is a redirection for a streaming request + crawler_config = CrawlerRunConfig.load(crawl_request.crawler_config) + if crawler_config.stream: + return await stream_process(crawl_request=crawl_request) results = await handle_crawl_request( urls=crawl_request.urls, browser_config=crawl_request.browser_config, @@ -506,12 +511,16 @@ async def crawl_stream( ): if not crawl_request.urls: raise HTTPException(400, "At least one URL required") + + return await stream_process(crawl_request=crawl_request) + +async def stream_process(crawl_request: CrawlRequest): crawler, gen = await handle_stream_crawl_request( urls=crawl_request.urls, browser_config=crawl_request.browser_config, crawler_config=crawl_request.crawler_config, config=config, - ) +) return StreamingResponse( stream_results(crawler, gen), media_type="application/x-ndjson", diff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.html index e01fc857..31e10368 100644 --- a/deploy/docker/static/playground/index.html +++ b/deploy/docker/static/playground/index.html @@ -182,8 +182,8 @@

Request Builder

@@ -371,7 +371,7 @@
- + + +
+
+
+ HTML + CSS + +
+ 
<button class="brand-btn brand-btn-primary">
+  Launch Editor →
+</button>
+
+
+ +
+

Secondary Button

+
+ + +
+
+
+ HTML + CSS + +
+ 
<button class="brand-btn brand-btn-secondary">
+  View Documentation
+</button>
+
+
+ +
+

Accent Button

+
+ + +
+
+
+ HTML + CSS + +
+ 
<button class="brand-btn brand-btn-accent">
+  Try Beta Features
+</button>
+
+
+ +
+

Ghost Button

+
+ + +
+
+
+ HTML + CSS + +
+ 
<button class="brand-btn brand-btn-ghost">
+  Learn More
+</button>
+
+
+ + +### Badges & Status Indicators + +
+

Status Badges

+
+ Available + Beta + Alpha + New! + Coming Soon +
+
+
+ HTML + CSS + +
+ 
<span class="brand-badge badge-available">Available</span>
+<span class="brand-badge badge-beta">Beta</span>
+<span class="brand-badge badge-alpha">Alpha</span>
+<span class="brand-badge badge-new">New!</span>
+
+
+ +### Cards + +
+
+

🎨 C4A-Script Editor

+

A visual, block-based programming environment for creating browser automation scripts. Perfect for beginners and experts alike!

+ +
+ +
+

🧠 LLM Context Builder

+

Generate optimized context files for your favorite LLM when working with Crawl4AI. Get focused, relevant documentation based on your needs.

+ +
+
+ +
+
+ HTML + CSS + +
+ 
<div class="brand-card">
+  <h3 class="brand-card-title">Card Title</h3>
+  <p class="brand-card-description">Card description...</p>
+</div>
+
+ +### Terminal Window + +
+
+
+ + + +
+ crawl4ai@terminal ~ % +
+
+

$ pip install crawl4ai

+

Successfully installed crawl4ai-0.7.2

+
+
+ +
+
+ HTML + CSS + +
+ 
<div class="terminal-window">
+  <div class="terminal-header">
+    <div class="terminal-dots">
+      <span class="terminal-dot red"></span>
+      <span class="terminal-dot yellow"></span>
+      <span class="terminal-dot green"></span>
+    </div>
+    <span class="terminal-title">Terminal Title</span>
+  </div>
+  <div class="terminal-content">
+    Your content here
+  </div>
+</div>
+
+ +--- + +
+ 📐 +

Spacing & Layout

+
+ +### Spacing System + +Our spacing system is based on multiples of **10px** for consistency and ease of calculation. + +
+
+
+ 10px - Extra Small (xs) +
+
+
+ 20px - Small (sm) +
+
+
+ 30px - Medium (md) +
+
+
+ 40px - Large (lg) +
+
+
+ 60px - Extra Large (xl) +
+
+
+ 80px - 2XL +
+
+ +### Layout Patterns + +#### Terminal Container +Full-height, flex-column layout with sticky header + +```css +.terminal-container { + min-height: 100vh; + display: flex; + flex-direction: column; +} +``` + +#### Content Grid +Auto-fit responsive grid for cards and components + +```css +.component-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); + gap: 2rem; +} +``` + +#### Centered Content +Maximum width with auto margins for centered layouts + +```css +.content { + max-width: 1200px; + margin: 0 auto; + padding: 2rem; +} +``` + +--- + +
+ ✅ +

Usage Guidelines

+
+ +### When to Use Each Style + +**Documentation Pages (`docs/md_v2/core`, `/advanced`, etc.)** +- Use main documentation styles from `styles.css` and `layout.css` +- Terminal theme with sidebar navigation +- Dense, informative content +- ToC on the right side +- Focus on readability and technical accuracy + +**Landing Pages (`docs/md_v2/apps/crawl4ai-assistant`, etc.)** +- Use `assistant.css` style approach +- Hero sections with gradients +- Feature cards with hover effects +- Video/demo sections +- Sticky header with navigation +- Marketing-focused, visually engaging + +**App Home (`docs/md_v2/apps/index.md`)** +- Grid-based card layouts +- Status badges +- Call-to-action buttons +- Feature highlights +- Mix of informational and promotional + +**Interactive Apps (`docs/md_v2/apps/llmtxt`, `/c4a-script`)** +- Full-screen application layouts +- Interactive controls +- Real-time feedback +- Tool-specific UI patterns +- Functional over decorative + +**Chrome Extension (`popup.css`)** +- Compact, fixed-width design (380px) +- Clear mode selection +- Session indicators +- Minimal but effective +- Fast loading, no heavy assets + +### Do's and Don'ts + +
+
+
✅ DO
+
+ +
+
+ Use primary cyan for main CTAs and important actions +
+
+ +
+
❌ DON'T
+
+ +
+
+ Don't use arbitrary colors not in the brand palette +
+
+
+ +
+
+
✅ DO
+
+
+ async with AsyncWebCrawler(): +
+
+
+ Use Dank Mono for all text to maintain terminal aesthetic +
+
+ +
+
❌ DON'T
+
+
+ async with AsyncWebCrawler(): +
+
+
+ Don't use non-monospace fonts (breaks terminal feel) +
+
+
+ +
+
+
✅ DO
+
+
+ Beta +

New Feature

+
+
+
+ Use status badges to indicate feature maturity +
+
+ +
+
❌ DON'T
+
+
+

New Feature (Beta)

+
+
+
+ Don't put status indicators in plain text +
+
+
+ +--- + +
+ 🎯 +

Accessibility

+
+ +### Color Contrast + +All color combinations meet WCAG AA standards: + +- **Primary Cyan (#50ffff) on Dark (#070708)**: 12.4:1 ✅ +- **Primary Text (#e8e9ed) on Dark (#070708)**: 11.8:1 ✅ +- **Secondary Text (#d5cec0) on Dark (#070708)**: 9.2:1 ✅ +- **Tertiary Text (#a3abba) on Dark (#070708)**: 6.8:1 ✅ + +### Focus States + +All interactive elements must have visible focus indicators: + +```css +button:focus, +a:focus { + outline: 2px solid #50ffff; + outline-offset: 2px; +} +``` + +### Motion + +Respect `prefers-reduced-motion` for users who need it: + +```css +@media (prefers-reduced-motion: reduce) { + * { + animation-duration: 0.01ms !important; + transition-duration: 0.01ms !important; + } +} +``` + +--- + +
+ 💾 +

CSS Variables

+
+ +Use these CSS variables for consistency across all styles: + +```css +:root { + /* Colors */ + --primary-color: #50ffff; + --primary-dimmed: #09b5a5; + --primary-green: #0fbbaa; + --accent-color: #f380f5; + + /* Backgrounds */ + --background-color: #070708; + --bg-secondary: #1a1a1a; + --code-bg-color: #3f3f44; + --border-color: #3f3f44; + + /* Text */ + --font-color: #e8e9ed; + --secondary-color: #d5cec0; + --tertiary-color: #a3abba; + + /* Semantic */ + --success-color: #50ff50; + --error-color: #ff3c74; + --warning-color: #f59e0b; + + /* Typography */ + --font-primary: 'Dank Mono', dm, Monaco, Courier New, monospace; + --global-font-size: 14px; + --global-line-height: 1.6; + + /* Spacing */ + --global-space: 10px; + + /* Layout */ + --header-height: 65px; + --sidebar-width: 280px; + --toc-width: 340px; + --content-max-width: 90em; +} +``` + +--- + +
+ 📚 +

Resources

+
+ +### Download Assets + +- [Dank Mono Font Files](/docs/md_v2/assets/) (Regular, Bold, Italic) +- [Brand CSS Template](/docs/md_v2/branding/assets/brand-examples.css) +- [Component Library](/docs/md_v2/apps/) + +### Reference Files + +- Main Documentation Styles: `docs/md_v2/assets/styles.css` +- Layout System: `docs/md_v2/assets/layout.css` +- Landing Page Style: `docs/md_v2/apps/crawl4ai-assistant/assistant.css` +- App Home Style: `docs/md_v2/apps/index.md` +- Extension Style: `docs/md_v2/apps/crawl4ai-assistant/popup/popup.css` + +### Questions? + +If you're unsure about which style to use or need help implementing these guidelines: + +- Check existing examples in the relevant section +- Review the "When to Use Each Style" guidelines above +- Ask in our [Discord community](https://discord.gg/crawl4ai) +- Open an issue on [GitHub](https://github.com/unclecode/crawl4ai) + +--- + +
+

🎨 Keep It Terminal

+

+ When in doubt, ask yourself: "Does this feel like a terminal?" If yes, you're on brand. +

+
+ + diff --git a/mkdocs.yml b/mkdocs.yml index ff148547..50f19fce 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,4 @@ site_name: Crawl4AI Documentation (v0.7.x) -site_favicon: docs/md_v2/favicon.ico site_description: 🚀🤖 Crawl4AI, Open-source LLM-Friendly Web Crawler & Scraper site_url: https://docs.crawl4ai.com repo_url: https://github.com/unclecode/crawl4ai @@ -66,10 +65,12 @@ nav: - "CrawlResult": "api/crawl-result.md" - "Strategies": "api/strategies.md" - "C4A-Script Reference": "api/c4a-script-reference.md" + - "Brand Book": "branding/index.md" theme: name: 'terminal' palette: 'dark' + favicon: favicon.ico custom_dir: docs/md_v2/overrides color_mode: 'dark' icon: @@ -98,6 +99,7 @@ extra_css: - assets/highlight.css - assets/dmvendor.css - assets/feedback-overrides.css + - assets/page_actions.css extra_javascript: - https://www.googletagmanager.com/gtag/js?id=G-58W0K2ZQ25 @@ -106,8 +108,9 @@ extra_javascript: - assets/highlight_init.js - https://buttons.github.io/buttons.js - assets/toc.js - - assets/github_stats.js + - assets/github_stats.js - assets/selection_ask_ai.js - assets/copy_code.js - assets/floating_ask_ai_button.js - - assets/mobile_menu.js \ No newline at end of file + - assets/mobile_menu.js + - assets/page_actions.js \ No newline at end of file From ef46df10da35093ba70a491e62646d7cbecb2572 Mon Sep 17 00:00:00 2001 From: unclecode Date: Tue, 30 Sep 2025 18:31:57 +0800 Subject: [PATCH 039/119] Update gitignore add local scripts folder --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 03c5b355..710e1340 100644 --- a/.gitignore +++ b/.gitignore @@ -270,4 +270,6 @@ docs/**/data .codecat/ docs/apps/linkdin/debug*/ -docs/apps/linkdin/samples/insights/* \ No newline at end of file +docs/apps/linkdin/samples/insights/* + +scripts/ From 8d3066264755f4c275a5fb1d52dbaf581188dd61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Sj=C3=B6borg?= Date: Thu, 2 Oct 2025 09:17:32 +0200 Subject: [PATCH 040/119] fix: remove this import as it causes python to treat "json" as a variable in the except block --- deploy/docker/api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 78a36bf3..d0127e7b 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -563,7 +563,6 @@ async def handle_crawl_request( if isinstance(hook_manager, UserHookManager): try: # Ensure all hook data is JSON serializable - import json hook_data = { "status": hooks_status, "execution_log": hook_manager.execution_log, From 35dd206925e2e189e2338e654a55ed4a3cd2d918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Sj=C3=B6borg?= Date: Thu, 2 Oct 2025 09:20:59 +0200 Subject: [PATCH 041/119] fix: always return a list, even if we catch an exception --- crawl4ai/async_dispatcher.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crawl4ai/async_dispatcher.py b/crawl4ai/async_dispatcher.py index 5bb1a47c..bd44557c 100644 --- a/crawl4ai/async_dispatcher.py +++ b/crawl4ai/async_dispatcher.py @@ -455,8 +455,6 @@ class MemoryAdaptiveDispatcher(BaseDispatcher): # Update priorities for waiting tasks if needed await self._update_queue_priorities() - - return results except Exception as e: if self.monitor: @@ -467,6 +465,7 @@ class MemoryAdaptiveDispatcher(BaseDispatcher): memory_monitor.cancel() if self.monitor: self.monitor.stop() + return results async def _update_queue_priorities(self): """Periodically update priorities of items in the queue to prevent starvation""" From 408ad1b750b054c0a1b1a0801b008ee7d3406dd2 Mon Sep 17 00:00:00 2001 From: unclecode Date: Thu, 2 Oct 2025 16:41:11 +0800 Subject: [PATCH 042/119] feat(marketplace): Add Crawl4AI marketplace with secure configuration - Implement marketplace frontend and admin dashboard - Add FastAPI backend with environment-based configuration - Use .env file for secrets management - Include data generation scripts - Add proper CORS configuration - Remove hardcoded password from admin login - Update gitignore for security --- .gitignore | 7 + docs/md_v2/assets/images/logo.png | Bin 0 -> 1622 bytes docs/md_v2/marketplace/README.md | 66 ++ docs/md_v2/marketplace/admin/admin.css | 650 ++++++++++++ docs/md_v2/marketplace/admin/admin.js | 757 ++++++++++++++ docs/md_v2/marketplace/admin/index.html | 215 ++++ docs/md_v2/marketplace/backend/.env.example | 14 + docs/md_v2/marketplace/backend/config.py | 59 ++ docs/md_v2/marketplace/backend/database.py | 117 +++ docs/md_v2/marketplace/backend/dummy_data.py | 267 +++++ .../marketplace/backend/requirements.txt | 5 + docs/md_v2/marketplace/backend/schema.yaml | 75 ++ docs/md_v2/marketplace/backend/server.py | 390 +++++++ .../md_v2/marketplace/frontend/app-detail.css | 462 +++++++++ .../marketplace/frontend/app-detail.html | 234 +++++ docs/md_v2/marketplace/frontend/app-detail.js | 324 ++++++ docs/md_v2/marketplace/frontend/index.html | 147 +++ .../marketplace/frontend/marketplace.css | 957 ++++++++++++++++++ .../md_v2/marketplace/frontend/marketplace.js | 395 ++++++++ mkdocs.yml | 2 + 20 files changed, 5143 insertions(+) create mode 100644 docs/md_v2/assets/images/logo.png create mode 100644 docs/md_v2/marketplace/README.md create mode 100644 docs/md_v2/marketplace/admin/admin.css create mode 100644 docs/md_v2/marketplace/admin/admin.js create mode 100644 docs/md_v2/marketplace/admin/index.html create mode 100644 docs/md_v2/marketplace/backend/.env.example create mode 100644 docs/md_v2/marketplace/backend/config.py create mode 100644 docs/md_v2/marketplace/backend/database.py create mode 100644 docs/md_v2/marketplace/backend/dummy_data.py create mode 100644 docs/md_v2/marketplace/backend/requirements.txt create mode 100644 docs/md_v2/marketplace/backend/schema.yaml create mode 100644 docs/md_v2/marketplace/backend/server.py create mode 100644 docs/md_v2/marketplace/frontend/app-detail.css create mode 100644 docs/md_v2/marketplace/frontend/app-detail.html create mode 100644 docs/md_v2/marketplace/frontend/app-detail.js create mode 100644 docs/md_v2/marketplace/frontend/index.html create mode 100644 docs/md_v2/marketplace/frontend/marketplace.css create mode 100644 docs/md_v2/marketplace/frontend/marketplace.js diff --git a/.gitignore b/.gitignore index 710e1340..a5389a3e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,13 @@ # Scripts folder (private tools) .scripts/ +# Database files +*.db + +# Environment files +.env +.env.local + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/docs/md_v2/assets/images/logo.png b/docs/md_v2/assets/images/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..ed82a3ccde17d44c7ed43c91687284894250eb42 GIT binary patch literal 1622 zcmV-c2C4apP)JU12>}FcPzs4AiaugUcoPT&g$FMBfPuuQTw96=z1kLO0f{Z; zl3wJryPprUr)STeJ*7;t+1=TH{_mS_{`u!$p|vJ5O(3|@^NnNJJ#Ab7h8z>rPb}k* z`R0L56nM@d@HEgMhvQXX94(i?xEzmTTF=nJh5dLtgR~z@j-WtcpsCF7%Gf(uKq)=o z{(Av%nITNqS8xj^AS4ip0*(Waz`&sC%bB%5yglGTm@x!kJ3u&M`Uv=C@DXqa52xX5 z-wyldv#?K`0@ths`ul-U2xD&pVQl*Q!FS$;ec~kSGiSk?8X%o^mCSA7+*i7s1mxbm zaOTbfQzg;eBfF2gtm0< zbc6&VVW58im^cy6p~K({C4gf?ZrciZ^eC820wW{7skq=PSHk)F8$e2kjt+!YzXkU6 z0P#35JnY7UM@4OEj$Uz#IwYw9Y9XjYA?lJ67M;65)peVy^CtE3<+K3mWRlvl%(!Hd z7Qonw6;z!!sk*vZbp8T$NlDfpKrJP8JnrI|u~`L#1fDzr=g$YKs(|iph*Kv`FqM_T zK6nV2Fab!X5&H0b;Ph!AnKVNmJpz|5h5hYbAQAz3{zYi*8lbruNF=~F-UOb14mfuX z;{17_NSOf7Q6L_tjz+0x&ZN5DNHs9T*v?(F!Xa7$)RmP~*RE6D>ZWRKqpq%|28
{Q2l+2>e`>wsg&uDgjw{}ZmNMHs>WvO@^XVQs)A;gfY#K-#YU>qQmVRos=kMe zZU2;(l*ZwlIaH0!hD38Kb!8=Wc{$aMCaPOq24wDBV=E0qf zyX>AA$gF3=7|xnyzE@7Tu*iBvUzT6r(V2x)GoGl8>78_ zRLw2abLJYK$wUsgOTc122@_O$;XUtw}b+sF`2_KHR!ugrZ(^Qd3OaS0`vfSgxZVp@{nCBZ!)Kyj4CD`&2Rc{|vODpxfd07>7 z-V%5^62-=_qQZdm-e>HS8d^fol7c#wqH6fVta(ctbyXGh?AcT;Z6;4#y+&P8;o3{a zcGNO@{{dCQRq7ct#tK41e|_(2~}&G z*&X#)Xic9^OA1=y5M!TzLDfHKlt`thqY;BuC=p@dLa_WLh`V(I~{7yO3*F1MM9UmIY_oGH}u)h`Kt6ix>Tg z$g>OL7PS;6!DLeL)Tvaz{Z7?=o2tE&vDcOrOvU>36;vHJsk&~n=$Buqr%a&^hpB}( z6jhG3jBzva*4bN5KEL;g)2|E$gCDQ`#|K0hfT^k;^HQ z#iT3%9z?)XlsXyL{)#yJBC~_McjTr}$6JDVfr6ehsoO(1FCBCADo~b>=Z7PxLJs=w zmp#{B7W`|>QDrPbegFpa6}Fd!K4c3=|N8oEgT5Df_uG#WC^W-o`4xT|nc$`RAG*CS UCSNaJPyhe`07*qoM6N<$f)VKA_5c6? literal 0 HcmV?d00001 diff --git a/docs/md_v2/marketplace/README.md b/docs/md_v2/marketplace/README.md new file mode 100644 index 00000000..75e1b5c6 --- /dev/null +++ b/docs/md_v2/marketplace/README.md @@ -0,0 +1,66 @@ +# Crawl4AI Marketplace + +A terminal-themed marketplace for tools, integrations, and resources related to Crawl4AI. + +## Setup + +### Backend + +1. Install dependencies: +```bash +cd backend +pip install -r requirements.txt +``` + +2. Generate dummy data: +```bash +python dummy_data.py +``` + +3. Run the server: +```bash +python server.py +``` + +The API will be available at http://localhost:8100 + +### Frontend + +1. Open `frontend/index.html` in your browser +2. Or serve via MkDocs as part of the documentation site + +## Database Schema + +The marketplace uses SQLite with automatic migration from `schema.yaml`. Tables include: +- **apps**: Tools and integrations +- **articles**: Reviews, tutorials, and news +- **categories**: App categories +- **sponsors**: Sponsored content + +## API Endpoints + +- `GET /api/apps` - List apps with filters +- `GET /api/articles` - List articles +- `GET /api/categories` - Get all categories +- `GET /api/sponsors` - Get active sponsors +- `GET /api/search?q=query` - Search across content +- `GET /api/stats` - Marketplace statistics + +## Features + +- **Smart caching**: LocalStorage with TTL (1 hour) +- **Terminal theme**: Consistent with Crawl4AI branding +- **Responsive design**: Works on all devices +- **Fast search**: Debounced with 300ms delay +- **CORS protected**: Only crawl4ai.com and localhost + +## Admin Panel + +Coming soon - for now, edit the database directly or modify `dummy_data.py` + +## Deployment + +For production deployment on EC2: +1. Update `API_BASE` in `marketplace.js` to production URL +2. Run FastAPI with proper production settings (use gunicorn/uvicorn) +3. Set up nginx proxy if needed \ No newline at end of file diff --git a/docs/md_v2/marketplace/admin/admin.css b/docs/md_v2/marketplace/admin/admin.css new file mode 100644 index 00000000..7296a801 --- /dev/null +++ b/docs/md_v2/marketplace/admin/admin.css @@ -0,0 +1,650 @@ +/* Admin Dashboard - C4AI Terminal Style */ + +/* Utility Classes */ +.hidden { + display: none !important; +} + +/* Brand Colors */ +:root { + --c4ai-cyan: #50ffff; + --c4ai-green: #50ff50; + --c4ai-yellow: #ffff50; + --c4ai-pink: #ff50ff; + --c4ai-blue: #5050ff; +} + +.admin-container { + min-height: 100vh; + background: var(--bg-dark); +} + +/* Login Screen */ +.login-screen { + min-height: 100vh; + display: flex; + align-items: center; + justify-content: center; + background: linear-gradient(135deg, #070708 0%, #1a1a2e 100%); +} + +.login-box { + background: var(--bg-secondary); + border: 2px solid var(--primary-cyan); + padding: 3rem; + width: 400px; + box-shadow: 0 0 40px rgba(80, 255, 255, 0.2); + text-align: center; +} + +.login-logo { + height: 60px; + margin-bottom: 2rem; + filter: brightness(1.2); +} + +.login-box h1 { + color: var(--primary-cyan); + font-size: 1.5rem; + margin-bottom: 2rem; +} + +#login-form input { + width: 100%; + padding: 0.75rem; + background: var(--bg-dark); + border: 1px solid var(--border-color); + color: var(--text-primary); + font-family: inherit; + margin-bottom: 1rem; +} + +#login-form input:focus { + outline: none; + border-color: var(--primary-cyan); +} + +#login-form button { + width: 100%; + padding: 0.75rem; + background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal)); + border: none; + color: var(--bg-dark); + font-weight: 600; + cursor: pointer; + transition: all 0.2s; +} + +#login-form button:hover { + box-shadow: 0 4px 15px rgba(80, 255, 255, 0.3); + transform: translateY(-2px); +} + +.error-msg { + color: var(--error); + font-size: 0.875rem; + margin-top: 1rem; +} + +/* Admin Dashboard */ +.admin-dashboard.hidden { + display: none; +} + +.admin-header { + background: var(--bg-secondary); + border-bottom: 2px solid var(--primary-cyan); + padding: 1rem 0; +} + +.header-content { + max-width: 1800px; + margin: 0 auto; + padding: 0 2rem; + display: flex; + justify-content: space-between; + align-items: center; +} + +.header-left { + display: flex; + align-items: center; + gap: 1rem; +} + +.header-logo { + height: 35px; +} + +.admin-header h1 { + font-size: 1.25rem; + color: var(--primary-cyan); +} + +.header-right { + display: flex; + align-items: center; + gap: 2rem; +} + +.admin-user { + color: var(--text-secondary); +} + +.logout-btn { + padding: 0.5rem 1rem; + background: transparent; + border: 1px solid var(--error); + color: var(--error); + cursor: pointer; + transition: all 0.2s; +} + +.logout-btn:hover { + background: rgba(255, 60, 116, 0.1); +} + +/* Layout */ +.admin-layout { + display: flex; + max-width: 1800px; + margin: 0 auto; + min-height: calc(100vh - 60px); +} + +/* Sidebar */ +.admin-sidebar { + width: 250px; + background: var(--bg-secondary); + border-right: 1px solid var(--border-color); + display: flex; + flex-direction: column; + justify-content: space-between; +} + +.sidebar-nav { + padding: 1rem 0; +} + +.nav-btn { + width: 100%; + padding: 1rem 1.5rem; + background: transparent; + border: none; + border-left: 3px solid transparent; + color: var(--text-secondary); + text-align: left; + cursor: pointer; + transition: all 0.2s; + display: flex; + align-items: center; + gap: 0.75rem; +} + +.nav-btn:hover { + background: rgba(80, 255, 255, 0.05); + color: var(--primary-cyan); +} + +.nav-btn.active { + border-left-color: var(--primary-cyan); + background: rgba(80, 255, 255, 0.1); + color: var(--primary-cyan); +} + +.nav-icon { + font-size: 1.25rem; + margin-right: 0.25rem; + display: inline-block; + width: 1.5rem; + text-align: center; +} + +.nav-btn[data-section="stats"] .nav-icon { + color: var(--c4ai-cyan); +} + +.nav-btn[data-section="apps"] .nav-icon { + color: var(--c4ai-green); +} + +.nav-btn[data-section="articles"] .nav-icon { + color: var(--c4ai-yellow); +} + +.nav-btn[data-section="categories"] .nav-icon { + color: var(--c4ai-pink); +} + +.nav-btn[data-section="sponsors"] .nav-icon { + color: var(--c4ai-blue); +} + +.sidebar-actions { + padding: 1rem; + border-top: 1px solid var(--border-color); +} + +.action-btn { + width: 100%; + padding: 0.75rem; + background: var(--bg-tertiary); + border: 1px solid var(--border-color); + color: var(--text-secondary); + cursor: pointer; + margin-bottom: 0.5rem; + transition: all 0.2s; +} + +.action-btn:hover { + border-color: var(--primary-cyan); + color: var(--primary-cyan); +} + +/* Main Content */ +.admin-main { + flex: 1; + padding: 2rem; + overflow-y: auto; +} + +.content-section { + display: none; +} + +.content-section.active { + display: block; +} + +/* Stats Grid */ +.stats-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + gap: 1.5rem; + margin-bottom: 3rem; +} + +.stat-card { + background: linear-gradient(135deg, rgba(80, 255, 255, 0.03), rgba(243, 128, 245, 0.02)); + border: 1px solid rgba(80, 255, 255, 0.3); + padding: 1.5rem; + display: flex; + gap: 1.5rem; +} + +.stat-icon { + font-size: 2rem; + width: 3rem; + height: 3rem; + display: flex; + align-items: center; + justify-content: center; + border: 2px solid; + border-radius: 4px; +} + +.stat-card:nth-child(1) .stat-icon { + color: var(--c4ai-cyan); + border-color: var(--c4ai-cyan); +} + +.stat-card:nth-child(2) .stat-icon { + color: var(--c4ai-green); + border-color: var(--c4ai-green); +} + +.stat-card:nth-child(3) .stat-icon { + color: var(--c4ai-yellow); + border-color: var(--c4ai-yellow); +} + +.stat-card:nth-child(4) .stat-icon { + color: var(--c4ai-pink); + border-color: var(--c4ai-pink); +} + +.stat-number { + font-size: 2rem; + color: var(--primary-cyan); + font-weight: 600; +} + +.stat-label { + color: var(--text-secondary); +} + +.stat-detail { + font-size: 0.875rem; + color: var(--text-tertiary); + margin-top: 0.5rem; +} + +/* Quick Actions */ +.quick-actions { + display: flex; + gap: 1rem; +} + +.quick-btn { + padding: 0.75rem 1.5rem; + background: transparent; + border: 1px solid var(--primary-cyan); + color: var(--primary-cyan); + cursor: pointer; + transition: all 0.2s; +} + +.quick-btn:hover { + background: rgba(80, 255, 255, 0.1); + transform: translateY(-2px); +} + +/* Section Headers */ +.section-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 2rem; +} + +.section-header h2 { + font-size: 1.5rem; + color: var(--text-primary); +} + +.header-actions { + display: flex; + gap: 1rem; +} + +.search-input { + padding: 0.5rem 1rem; + background: var(--bg-dark); + border: 1px solid var(--border-color); + color: var(--text-primary); + width: 250px; +} + +.search-input:focus { + outline: none; + border-color: var(--primary-cyan); +} + +.filter-select { + padding: 0.5rem; + background: var(--bg-dark); + border: 1px solid var(--border-color); + color: var(--text-primary); +} + +.add-btn { + padding: 0.5rem 1rem; + background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal)); + border: none; + color: var(--bg-dark); + font-weight: 600; + cursor: pointer; + transition: all 0.2s; +} + +.add-btn:hover { + box-shadow: 0 4px 15px rgba(80, 255, 255, 0.3); + transform: translateY(-2px); +} + +/* Data Tables */ +.data-table { + background: var(--bg-secondary); + border: 1px solid var(--border-color); + overflow-x: auto; +} + +.data-table table { + width: 100%; + border-collapse: collapse; +} + +.data-table th { + background: var(--bg-tertiary); + padding: 1rem; + text-align: left; + color: var(--primary-cyan); + font-weight: 600; + border-bottom: 2px solid var(--border-color); + position: sticky; + top: 0; + z-index: 10; +} + +.data-table td { + padding: 1rem; + border-bottom: 1px solid var(--border-color); +} + +.data-table tr:hover { + background: rgba(80, 255, 255, 0.03); +} + +/* Table Actions */ +.table-actions { + display: flex; + gap: 0.5rem; +} + +.btn-edit, .btn-delete, .btn-duplicate { + padding: 0.25rem 0.5rem; + background: transparent; + border: 1px solid var(--border-color); + color: var(--text-secondary); + cursor: pointer; + font-size: 0.875rem; +} + +.btn-edit:hover { + border-color: var(--primary-cyan); + color: var(--primary-cyan); +} + +.btn-delete:hover { + border-color: var(--error); + color: var(--error); +} + +.btn-duplicate:hover { + border-color: var(--accent-pink); + color: var(--accent-pink); +} + +/* Badges in Tables */ +.badge { + padding: 0.25rem 0.5rem; + font-size: 0.75rem; + text-transform: uppercase; +} + +.badge.featured { + background: var(--primary-cyan); + color: var(--bg-dark); +} + +.badge.sponsored { + background: var(--warning); + color: var(--bg-dark); +} + +.badge.active { + background: var(--success); + color: var(--bg-dark); +} + +/* Modal Enhancements */ +.modal-content.large { + max-width: 1000px; + width: 90%; + max-height: 90vh; +} + +.modal-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 1.5rem; + border-bottom: 1px solid var(--border-color); +} + +.modal-body { + padding: 1.5rem; + overflow-y: auto; + max-height: calc(90vh - 140px); +} + +.modal-footer { + display: flex; + justify-content: flex-end; + gap: 1rem; + padding: 1rem 1.5rem; + border-top: 1px solid var(--border-color); +} + +.btn-cancel, .btn-save { + padding: 0.5rem 1.5rem; + cursor: pointer; + transition: all 0.2s; +} + +.btn-cancel { + background: transparent; + border: 1px solid var(--border-color); + color: var(--text-secondary); +} + +.btn-cancel:hover { + border-color: var(--error); + color: var(--error); +} + +.btn-save { + background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal)); + border: none; + color: var(--bg-dark); + font-weight: 600; +} + +.btn-save:hover { + box-shadow: 0 4px 15px rgba(80, 255, 255, 0.3); +} + +/* Form Styles */ +.form-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); + gap: 1.5rem; +} + +.form-group { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.form-group label { + color: var(--text-secondary); + font-size: 0.875rem; +} + +.form-group input, +.form-group select, +.form-group textarea { + padding: 0.5rem; + background: var(--bg-dark); + border: 1px solid var(--border-color); + color: var(--text-primary); + font-family: inherit; +} + +.form-group input:focus, +.form-group select:focus, +.form-group textarea:focus { + outline: none; + border-color: var(--primary-cyan); +} + +.form-group.full-width { + grid-column: 1 / -1; +} + +.checkbox-group { + display: flex; + gap: 2rem; +} + +.checkbox-label { + display: flex; + align-items: center; + gap: 0.5rem; + cursor: pointer; +} + +/* Rich Text Editor */ +.editor-toolbar { + display: flex; + gap: 0.5rem; + padding: 0.5rem; + background: var(--bg-tertiary); + border: 1px solid var(--border-color); + border-bottom: none; +} + +.editor-btn { + padding: 0.25rem 0.5rem; + background: transparent; + border: 1px solid var(--border-color); + color: var(--text-secondary); + cursor: pointer; +} + +.editor-btn:hover { + background: rgba(80, 255, 255, 0.1); + border-color: var(--primary-cyan); +} + +.editor-content { + min-height: 300px; + padding: 1rem; + background: var(--bg-dark); + border: 1px solid var(--border-color); + font-family: 'Dank Mono', Monaco, monospace; +} + +/* Responsive */ +@media (max-width: 1024px) { + .admin-layout { + flex-direction: column; + } + + .admin-sidebar { + width: 100%; + border-right: none; + border-bottom: 1px solid var(--border-color); + } + + .sidebar-nav { + display: flex; + overflow-x: auto; + padding: 0; + } + + .nav-btn { + border-left: none; + border-bottom: 3px solid transparent; + white-space: nowrap; + } + + .nav-btn.active { + border-bottom-color: var(--primary-cyan); + } + + .sidebar-actions { + display: none; + } +} \ No newline at end of file diff --git a/docs/md_v2/marketplace/admin/admin.js b/docs/md_v2/marketplace/admin/admin.js new file mode 100644 index 00000000..861d3ba7 --- /dev/null +++ b/docs/md_v2/marketplace/admin/admin.js @@ -0,0 +1,757 @@ +// Admin Dashboard - Smart & Powerful +const API_BASE = 'http://localhost:8100/api'; + +class AdminDashboard { + constructor() { + this.token = localStorage.getItem('admin_token'); + this.currentSection = 'stats'; + this.data = { + apps: [], + articles: [], + categories: [], + sponsors: [] + }; + this.editingItem = null; + this.init(); + } + + async init() { + // Check auth + if (!this.token) { + this.showLogin(); + return; + } + + // Try to load stats to verify token + try { + await this.loadStats(); + this.showDashboard(); + this.setupEventListeners(); + await this.loadAllData(); + } catch (error) { + if (error.status === 401) { + this.showLogin(); + } + } + } + + showLogin() { + document.getElementById('login-screen').classList.remove('hidden'); + document.getElementById('admin-dashboard').classList.add('hidden'); + + // Set up login button click handler + const loginBtn = document.getElementById('login-btn'); + if (loginBtn) { + loginBtn.onclick = async () => { + const password = document.getElementById('password').value; + await this.login(password); + }; + } + } + + async login(password) { + try { + const response = await fetch(`${API_BASE}/admin/login`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ password }) + }); + + if (!response.ok) throw new Error('Invalid password'); + + const data = await response.json(); + this.token = data.token; + localStorage.setItem('admin_token', this.token); + + document.getElementById('login-screen').classList.add('hidden'); + this.showDashboard(); + this.setupEventListeners(); + await this.loadAllData(); + } catch (error) { + document.getElementById('login-error').textContent = 'Invalid password'; + document.getElementById('password').value = ''; + } + } + + showDashboard() { + document.getElementById('login-screen').classList.add('hidden'); + document.getElementById('admin-dashboard').classList.remove('hidden'); + } + + setupEventListeners() { + // Navigation + document.querySelectorAll('.nav-btn').forEach(btn => { + btn.onclick = () => this.switchSection(btn.dataset.section); + }); + + // Logout + document.getElementById('logout-btn').onclick = () => this.logout(); + + // Export/Backup + document.getElementById('export-btn').onclick = () => this.exportData(); + document.getElementById('backup-btn').onclick = () => this.backupDatabase(); + + // Search + ['apps', 'articles'].forEach(type => { + const searchInput = document.getElementById(`${type}-search`); + if (searchInput) { + searchInput.oninput = (e) => this.filterTable(type, e.target.value); + } + }); + + // Category filter + const categoryFilter = document.getElementById('apps-filter'); + if (categoryFilter) { + categoryFilter.onchange = (e) => this.filterByCategory(e.target.value); + } + + // Save button in modal + document.getElementById('save-btn').onclick = () => this.saveItem(); + } + + async loadAllData() { + try { + await this.loadStats(); + } catch (e) { + console.error('Failed to load stats:', e); + } + + try { + await this.loadApps(); + } catch (e) { + console.error('Failed to load apps:', e); + } + + try { + await this.loadArticles(); + } catch (e) { + console.error('Failed to load articles:', e); + } + + try { + await this.loadCategories(); + } catch (e) { + console.error('Failed to load categories:', e); + } + + try { + await this.loadSponsors(); + } catch (e) { + console.error('Failed to load sponsors:', e); + } + + this.populateCategoryFilter(); + } + + async apiCall(endpoint, options = {}) { + const response = await fetch(`${API_BASE}${endpoint}`, { + ...options, + headers: { + 'Authorization': `Bearer ${this.token}`, + 'Content-Type': 'application/json', + ...options.headers + } + }); + + if (response.status === 401) { + this.logout(); + throw { status: 401 }; + } + + if (!response.ok) throw new Error(`API Error: ${response.status}`); + return response.json(); + } + + async loadStats() { + const stats = await this.apiCall('/admin/stats'); + + document.getElementById('stat-apps').textContent = stats.apps.total; + document.getElementById('stat-featured').textContent = stats.apps.featured; + document.getElementById('stat-sponsored').textContent = stats.apps.sponsored; + document.getElementById('stat-articles').textContent = stats.articles; + document.getElementById('stat-sponsors').textContent = stats.sponsors.active; + document.getElementById('stat-views').textContent = this.formatNumber(stats.total_views); + } + + async loadApps() { + this.data.apps = await this.apiCall('/apps?limit=100'); + this.renderAppsTable(this.data.apps); + } + + async loadArticles() { + this.data.articles = await this.apiCall('/articles?limit=100'); + this.renderArticlesTable(this.data.articles); + } + + async loadCategories() { + this.data.categories = await this.apiCall('/categories'); + this.renderCategoriesTable(this.data.categories); + } + + async loadSponsors() { + this.data.sponsors = await this.apiCall('/sponsors'); + this.renderSponsorsTable(this.data.sponsors); + } + + renderAppsTable(apps) { + const table = document.getElementById('apps-table'); + table.innerHTML = ` + + + + + + + + + + + + + + + ${apps.map(app => ` + + + + + + + + + + + `).join('')} + +
IDNameCategoryTypeRatingDownloadsStatusActions
${app.id}${app.name}${app.category}${app.type}◆ ${app.rating}/5${this.formatNumber(app.downloads)} + ${app.featured ? 'Featured' : ''} + ${app.sponsored ? '' : ''} + +
+ + + +
+
+ `; + } + + renderArticlesTable(articles) { + const table = document.getElementById('articles-table'); + table.innerHTML = ` + + + + + + + + + + + + + + ${articles.map(article => ` + + + + + + + + + + `).join('')} + +
IDTitleCategoryAuthorPublishedViewsActions
${article.id}${article.title}${article.category}${article.author}${new Date(article.published_date).toLocaleDateString()}${this.formatNumber(article.views)} +
+ + + +
+
+ `; + } + + renderCategoriesTable(categories) { + const table = document.getElementById('categories-table'); + table.innerHTML = ` + + + + + + + + + + + + ${categories.map(cat => ` + + + + + + + + `).join('')} + +
OrderIconNameDescriptionActions
${cat.order_index}${cat.icon}${cat.name}${cat.description} +
+ + +
+
+ `; + } + + renderSponsorsTable(sponsors) { + const table = document.getElementById('sponsors-table'); + table.innerHTML = ` + + + + + + + + + + + + + + ${sponsors.map(sponsor => ` + + + + + + + + + + `).join('')} + +
IDCompanyTierStartEndStatusActions
${sponsor.id}${sponsor.company_name}${sponsor.tier}${new Date(sponsor.start_date).toLocaleDateString()}${new Date(sponsor.end_date).toLocaleDateString()}${sponsor.active ? 'Active' : 'Inactive'} +
+ + +
+
+ `; + } + + showAddForm(type) { + this.editingItem = null; + this.showModal(type, null); + } + + async editItem(type, id) { + const item = this.data[type].find(i => i.id === id); + if (item) { + this.editingItem = item; + this.showModal(type, item); + } + } + + async duplicateItem(type, id) { + const item = this.data[type].find(i => i.id === id); + if (item) { + const newItem = { ...item }; + delete newItem.id; + newItem.name = `${newItem.name || newItem.title} (Copy)`; + if (newItem.slug) newItem.slug = `${newItem.slug}-copy-${Date.now()}`; + + this.editingItem = null; + this.showModal(type, newItem); + } + } + + showModal(type, item) { + const modal = document.getElementById('form-modal'); + const title = document.getElementById('modal-title'); + const body = document.getElementById('modal-body'); + + title.textContent = item ? `Edit ${type.slice(0, -1)}` : `Add New ${type.slice(0, -1)}`; + + if (type === 'apps') { + body.innerHTML = this.getAppForm(item); + } else if (type === 'articles') { + body.innerHTML = this.getArticleForm(item); + } else if (type === 'categories') { + body.innerHTML = this.getCategoryForm(item); + } else if (type === 'sponsors') { + body.innerHTML = this.getSponsorForm(item); + } + + modal.classList.remove('hidden'); + modal.dataset.type = type; + } + + getAppForm(app) { + return ` +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ `; + } + + getArticleForm(article) { + return ` +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ `; + } + + getCategoryForm(category) { + return ` +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ `; + } + + getSponsorForm(sponsor) { + return ` +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+ `; + } + + async saveItem() { + const modal = document.getElementById('form-modal'); + const type = modal.dataset.type; + const data = this.collectFormData(type); + + try { + if (this.editingItem) { + await this.apiCall(`/admin/${type}/${this.editingItem.id}`, { + method: 'PUT', + body: JSON.stringify(data) + }); + } else { + await this.apiCall(`/admin/${type}`, { + method: 'POST', + body: JSON.stringify(data) + }); + } + + this.closeModal(); + await this[`load${type.charAt(0).toUpperCase() + type.slice(1)}`](); + await this.loadStats(); + } catch (error) { + alert('Error saving item: ' + error.message); + } + } + + collectFormData(type) { + const data = {}; + + if (type === 'apps') { + data.name = document.getElementById('form-name').value; + data.slug = document.getElementById('form-slug').value || this.generateSlug(data.name); + data.description = document.getElementById('form-description').value; + data.category = document.getElementById('form-category').value; + data.type = document.getElementById('form-type').value; + data.rating = parseFloat(document.getElementById('form-rating').value); + data.downloads = parseInt(document.getElementById('form-downloads').value); + data.image = document.getElementById('form-image').value; + data.website_url = document.getElementById('form-website').value; + data.github_url = document.getElementById('form-github').value; + data.pricing = document.getElementById('form-pricing').value; + data.contact_email = document.getElementById('form-email').value; + data.featured = document.getElementById('form-featured').checked ? 1 : 0; + data.sponsored = document.getElementById('form-sponsored').checked ? 1 : 0; + data.integration_guide = document.getElementById('form-integration').value; + } else if (type === 'articles') { + data.title = document.getElementById('form-title').value; + data.slug = this.generateSlug(data.title); + data.author = document.getElementById('form-author').value; + data.category = document.getElementById('form-category').value; + data.featured_image = document.getElementById('form-image').value; + data.content = document.getElementById('form-content').value; + } else if (type === 'categories') { + data.name = document.getElementById('form-name').value; + data.slug = this.generateSlug(data.name); + data.icon = document.getElementById('form-icon').value; + data.description = document.getElementById('form-description').value; + data.order_index = parseInt(document.getElementById('form-order').value); + } else if (type === 'sponsors') { + data.company_name = document.getElementById('form-name').value; + data.tier = document.getElementById('form-tier').value; + data.landing_url = document.getElementById('form-landing').value; + data.banner_url = document.getElementById('form-banner').value; + data.start_date = document.getElementById('form-start').value; + data.end_date = document.getElementById('form-end').value; + data.active = document.getElementById('form-active').checked ? 1 : 0; + } + + return data; + } + + async deleteItem(type, id) { + if (!confirm(`Are you sure you want to delete this ${type.slice(0, -1)}?`)) return; + + try { + await this.apiCall(`/admin/${type}/${id}`, { method: 'DELETE' }); + await this[`load${type.charAt(0).toUpperCase() + type.slice(1)}`](); + await this.loadStats(); + } catch (error) { + alert('Error deleting item: ' + error.message); + } + } + + async deleteCategory(id) { + const hasApps = this.data.apps.some(app => + app.category === this.data.categories.find(c => c.id === id)?.name + ); + + if (hasApps) { + alert('Cannot delete category with existing apps'); + return; + } + + await this.deleteItem('categories', id); + } + + closeModal() { + document.getElementById('form-modal').classList.add('hidden'); + this.editingItem = null; + } + + switchSection(section) { + // Update navigation + document.querySelectorAll('.nav-btn').forEach(btn => { + btn.classList.toggle('active', btn.dataset.section === section); + }); + + // Show section + document.querySelectorAll('.content-section').forEach(sec => { + sec.classList.remove('active'); + }); + document.getElementById(`${section}-section`).classList.add('active'); + + this.currentSection = section; + } + + filterTable(type, query) { + const items = this.data[type].filter(item => { + const searchText = Object.values(item).join(' ').toLowerCase(); + return searchText.includes(query.toLowerCase()); + }); + + if (type === 'apps') { + this.renderAppsTable(items); + } else if (type === 'articles') { + this.renderArticlesTable(items); + } + } + + filterByCategory(category) { + const apps = category + ? this.data.apps.filter(app => app.category === category) + : this.data.apps; + this.renderAppsTable(apps); + } + + populateCategoryFilter() { + const filter = document.getElementById('apps-filter'); + if (!filter) return; + + filter.innerHTML = ''; + this.data.categories.forEach(cat => { + filter.innerHTML += ``; + }); + } + + async exportData() { + const data = { + apps: this.data.apps, + articles: this.data.articles, + categories: this.data.categories, + sponsors: this.data.sponsors, + exported: new Date().toISOString() + }; + + const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `marketplace-export-${Date.now()}.json`; + a.click(); + } + + async backupDatabase() { + // In production, this would download the SQLite file + alert('Database backup would be implemented on the server side'); + } + + generateSlug(text) { + return text.toLowerCase() + .replace(/[^\w\s-]/g, '') + .replace(/\s+/g, '-') + .replace(/-+/g, '-') + .trim(); + } + + formatNumber(num) { + if (num >= 1000000) return (num / 1000000).toFixed(1) + 'M'; + if (num >= 1000) return (num / 1000).toFixed(1) + 'K'; + return num.toString(); + } + + logout() { + localStorage.removeItem('admin_token'); + this.token = null; + this.showLogin(); + } +} + +// Initialize +const admin = new AdminDashboard(); \ No newline at end of file diff --git a/docs/md_v2/marketplace/admin/index.html b/docs/md_v2/marketplace/admin/index.html new file mode 100644 index 00000000..3691c1eb --- /dev/null +++ b/docs/md_v2/marketplace/admin/index.html @@ -0,0 +1,215 @@ + + + + + + Admin Dashboard - Crawl4AI Marketplace + + + + +
+ + + + + + + + +
+ + + + \ No newline at end of file diff --git a/docs/md_v2/marketplace/backend/.env.example b/docs/md_v2/marketplace/backend/.env.example new file mode 100644 index 00000000..7d46f19c --- /dev/null +++ b/docs/md_v2/marketplace/backend/.env.example @@ -0,0 +1,14 @@ +# Marketplace Configuration +# Copy this to .env and update with your values + +# Admin password (required) +MARKETPLACE_ADMIN_PASSWORD=change_this_password + +# JWT secret key (required) - generate with: python3 -c "import secrets; print(secrets.token_urlsafe(32))" +MARKETPLACE_JWT_SECRET=change_this_to_a_secure_random_key + +# Database path (optional, defaults to ./marketplace.db) +MARKETPLACE_DB_PATH=./marketplace.db + +# Token expiry in hours (optional, defaults to 4) +MARKETPLACE_TOKEN_EXPIRY=4 \ No newline at end of file diff --git a/docs/md_v2/marketplace/backend/config.py b/docs/md_v2/marketplace/backend/config.py new file mode 100644 index 00000000..29bb55d6 --- /dev/null +++ b/docs/md_v2/marketplace/backend/config.py @@ -0,0 +1,59 @@ +""" +Marketplace Configuration - Loads from .env file +""" +import os +import sys +import hashlib +from pathlib import Path +from dotenv import load_dotenv + +# Load .env file +env_path = Path(__file__).parent / '.env' +if not env_path.exists(): + print("\n❌ ERROR: No .env file found!") + print("Please copy .env.example to .env and update with your values:") + print(f" cp {Path(__file__).parent}/.env.example {Path(__file__).parent}/.env") + print("\nThen edit .env with your secure values.") + sys.exit(1) + +load_dotenv(env_path) + +# Required environment variables +required_vars = ['MARKETPLACE_ADMIN_PASSWORD', 'MARKETPLACE_JWT_SECRET'] +missing_vars = [var for var in required_vars if not os.getenv(var)] + +if missing_vars: + print(f"\n❌ ERROR: Missing required environment variables: {', '.join(missing_vars)}") + print("Please check your .env file and ensure all required variables are set.") + sys.exit(1) + +class Config: + """Configuration loaded from environment variables""" + + # Admin authentication - hashed from password in .env + ADMIN_PASSWORD_HASH = hashlib.sha256( + os.getenv('MARKETPLACE_ADMIN_PASSWORD').encode() + ).hexdigest() + + # JWT secret for token generation + JWT_SECRET_KEY = os.getenv('MARKETPLACE_JWT_SECRET') + + # Database path + DATABASE_PATH = os.getenv('MARKETPLACE_DB_PATH', './marketplace.db') + + # Token expiry in hours + TOKEN_EXPIRY_HOURS = int(os.getenv('MARKETPLACE_TOKEN_EXPIRY', '4')) + + # CORS origins - hardcoded as they don't contain secrets + ALLOWED_ORIGINS = [ + "http://localhost:8000", + "http://localhost:8080", + "http://localhost:8100", + "http://127.0.0.1:8000", + "http://127.0.0.1:8080", + "http://127.0.0.1:8100", + "https://crawl4ai.com", + "https://www.crawl4ai.com", + "https://docs.crawl4ai.com", + "https://market.crawl4ai.com" + ] \ No newline at end of file diff --git a/docs/md_v2/marketplace/backend/database.py b/docs/md_v2/marketplace/backend/database.py new file mode 100644 index 00000000..8ccfbaf4 --- /dev/null +++ b/docs/md_v2/marketplace/backend/database.py @@ -0,0 +1,117 @@ +import sqlite3 +import yaml +import json +from pathlib import Path +from typing import Dict, List, Any + +class DatabaseManager: + def __init__(self, db_path=None, schema_path='schema.yaml'): + self.schema = self._load_schema(schema_path) + # Use provided path or fallback to schema default + self.db_path = db_path or self.schema['database']['name'] + self.conn = None + self._init_database() + + def _load_schema(self, path: str) -> Dict: + with open(path, 'r') as f: + return yaml.safe_load(f) + + def _init_database(self): + """Auto-create/migrate database from schema""" + self.conn = sqlite3.connect(self.db_path, check_same_thread=False) + self.conn.row_factory = sqlite3.Row + + for table_name, table_def in self.schema['tables'].items(): + self._create_or_update_table(table_name, table_def['columns']) + + def _create_or_update_table(self, table_name: str, columns: Dict): + cursor = self.conn.cursor() + + # Check if table exists + cursor.execute(f"SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table_name,)) + table_exists = cursor.fetchone() is not None + + if not table_exists: + # Create table + col_defs = [] + for col_name, col_spec in columns.items(): + col_def = f"{col_name} {col_spec['type']}" + if col_spec.get('primary'): + col_def += " PRIMARY KEY" + if col_spec.get('autoincrement'): + col_def += " AUTOINCREMENT" + if col_spec.get('unique'): + col_def += " UNIQUE" + if col_spec.get('required'): + col_def += " NOT NULL" + if 'default' in col_spec: + default = col_spec['default'] + if default == 'CURRENT_TIMESTAMP': + col_def += f" DEFAULT {default}" + elif isinstance(default, str): + col_def += f" DEFAULT '{default}'" + else: + col_def += f" DEFAULT {default}" + col_defs.append(col_def) + + create_sql = f"CREATE TABLE {table_name} ({', '.join(col_defs)})" + cursor.execute(create_sql) + else: + # Check for new columns and add them + cursor.execute(f"PRAGMA table_info({table_name})") + existing_columns = {row[1] for row in cursor.fetchall()} + + for col_name, col_spec in columns.items(): + if col_name not in existing_columns: + col_def = f"{col_spec['type']}" + if 'default' in col_spec: + default = col_spec['default'] + if default == 'CURRENT_TIMESTAMP': + col_def += f" DEFAULT {default}" + elif isinstance(default, str): + col_def += f" DEFAULT '{default}'" + else: + col_def += f" DEFAULT {default}" + + cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN {col_name} {col_def}") + + self.conn.commit() + + def get_all(self, table: str, limit: int = 100, offset: int = 0, where: str = None) -> List[Dict]: + cursor = self.conn.cursor() + query = f"SELECT * FROM {table}" + if where: + query += f" WHERE {where}" + query += f" LIMIT {limit} OFFSET {offset}" + + cursor.execute(query) + rows = cursor.fetchall() + return [dict(row) for row in rows] + + def search(self, query: str, tables: List[str] = None) -> Dict[str, List[Dict]]: + if not tables: + tables = list(self.schema['tables'].keys()) + + results = {} + cursor = self.conn.cursor() + + for table in tables: + # Search in text columns + columns = self.schema['tables'][table]['columns'] + text_cols = [col for col, spec in columns.items() + if spec['type'] == 'TEXT' and col != 'id'] + + if text_cols: + where_clause = ' OR '.join([f"{col} LIKE ?" for col in text_cols]) + params = [f'%{query}%'] * len(text_cols) + + cursor.execute(f"SELECT * FROM {table} WHERE {where_clause} LIMIT 10", params) + rows = cursor.fetchall() + if rows: + results[table] = [dict(row) for row in rows] + + return results + + def close(self): + if self.conn: + self.conn.close() \ No newline at end of file diff --git a/docs/md_v2/marketplace/backend/dummy_data.py b/docs/md_v2/marketplace/backend/dummy_data.py new file mode 100644 index 00000000..3e7f46f9 --- /dev/null +++ b/docs/md_v2/marketplace/backend/dummy_data.py @@ -0,0 +1,267 @@ +import sqlite3 +import json +import random +from datetime import datetime, timedelta +from database import DatabaseManager + +def generate_slug(text): + return text.lower().replace(' ', '-').replace('&', 'and') + +def generate_dummy_data(): + db = DatabaseManager() + conn = db.conn + cursor = conn.cursor() + + # Clear existing data + for table in ['apps', 'articles', 'categories', 'sponsors']: + cursor.execute(f"DELETE FROM {table}") + + # Categories + categories = [ + ("Browser Automation", "⚙", "Tools for browser automation and control"), + ("Proxy Services", "🔒", "Proxy providers and rotation services"), + ("LLM Integration", "🤖", "AI/LLM tools and integrations"), + ("Data Processing", "📊", "Data extraction and processing tools"), + ("Cloud Infrastructure", "☁", "Cloud browser and computing services"), + ("Developer Tools", "🛠", "Development and testing utilities") + ] + + for i, (name, icon, desc) in enumerate(categories): + cursor.execute(""" + INSERT INTO categories (name, slug, icon, description, order_index) + VALUES (?, ?, ?, ?, ?) + """, (name, generate_slug(name), icon, desc, i)) + + # Apps with real Unsplash images + apps_data = [ + # Browser Automation + ("Playwright Cloud", "Browser Automation", "Paid", True, True, + "Scalable browser automation in the cloud with Playwright", "https://playwright.cloud", + None, "$99/month starter", 4.8, 12500, + "https://images.unsplash.com/photo-1633356122544-f134324a6cee?w=800&h=400&fit=crop"), + + ("Selenium Grid Hub", "Browser Automation", "Freemium", False, False, + "Distributed Selenium grid for parallel testing", "https://seleniumhub.io", + "https://github.com/seleniumhub/grid", "Free - $299/month", 4.2, 8400, + "https://images.unsplash.com/photo-1555066931-4365d14bab8c?w=800&h=400&fit=crop"), + + ("Puppeteer Extra", "Browser Automation", "Open Source", True, False, + "Enhanced Puppeteer with stealth plugins and more", "https://puppeteer-extra.dev", + "https://github.com/berstend/puppeteer-extra", "Free", 4.6, 15200, + "https://images.unsplash.com/photo-1461749280684-dccba630e2f6?w=800&h=400&fit=crop"), + + # Proxy Services + ("BrightData", "Proxy Services", "Paid", True, True, + "Premium proxy network with 72M+ IPs worldwide", "https://brightdata.com", + None, "Starting $500/month", 4.7, 9800, + "https://images.unsplash.com/photo-1558494949-ef010cbdcc31?w=800&h=400&fit=crop"), + + ("SmartProxy", "Proxy Services", "Paid", False, True, + "Residential and datacenter proxies with rotation", "https://smartproxy.com", + None, "Starting $75/month", 4.3, 7600, + "https://images.unsplash.com/photo-1544197150-b99a580bb7a8?w=800&h=400&fit=crop"), + + ("ProxyMesh", "Proxy Services", "Freemium", False, False, + "Rotating proxy servers with sticky sessions", "https://proxymesh.com", + None, "$10-$50/month", 4.0, 4200, + "https://images.unsplash.com/photo-1451187580459-43490279c0fa?w=800&h=400&fit=crop"), + + # LLM Integration + ("LangChain Crawl", "LLM Integration", "Open Source", True, False, + "LangChain integration for Crawl4AI workflows", "https://langchain-crawl.dev", + "https://github.com/langchain/crawl", "Free", 4.5, 18900, + "https://images.unsplash.com/photo-1677442136019-21780ecad995?w=800&h=400&fit=crop"), + + ("GPT Scraper", "LLM Integration", "Freemium", False, False, + "Extract structured data using GPT models", "https://gptscraper.ai", + None, "Free - $99/month", 4.1, 5600, + "https://images.unsplash.com/photo-1655720828018-edd2daec9349?w=800&h=400&fit=crop"), + + ("Claude Extract", "LLM Integration", "Paid", True, True, + "Professional extraction using Claude AI", "https://claude-extract.com", + None, "$199/month", 4.9, 3200, + "https://images.unsplash.com/photo-1686191128892-3b09ad503b4f?w=800&h=400&fit=crop"), + + # Data Processing + ("DataMiner Pro", "Data Processing", "Paid", False, False, + "Advanced data extraction and transformation", "https://dataminer.pro", + None, "$149/month", 4.2, 6700, + "https://images.unsplash.com/photo-1551288049-bebda4e38f71?w=800&h=400&fit=crop"), + + ("ScraperAPI", "Data Processing", "Freemium", True, True, + "Simple API for web scraping with proxy rotation", "https://scraperapi.com", + None, "Free - $299/month", 4.6, 22300, + "https://images.unsplash.com/photo-1460925895917-afdab827c52f?w=800&h=400&fit=crop"), + + ("Apify", "Data Processing", "Freemium", False, False, + "Web scraping and automation platform", "https://apify.com", + None, "$49-$499/month", 4.4, 14500, + "https://images.unsplash.com/photo-1504639725590-34d0984388bd?w=800&h=400&fit=crop"), + + # Cloud Infrastructure + ("BrowserCloud", "Cloud Infrastructure", "Paid", True, True, + "Managed headless browsers in the cloud", "https://browsercloud.io", + None, "$199/month", 4.5, 8900, + "https://images.unsplash.com/photo-1667372393119-3d4c48d07fc9?w=800&h=400&fit=crop"), + + ("LambdaTest", "Cloud Infrastructure", "Freemium", False, False, + "Cross-browser testing on cloud", "https://lambdatest.com", + None, "Free - $99/month", 4.1, 11200, + "https://images.unsplash.com/photo-1451187580459-43490279c0fa?w=800&h=400&fit=crop"), + + ("Browserless", "Cloud Infrastructure", "Freemium", True, False, + "Headless browser automation API", "https://browserless.io", + None, "$50-$500/month", 4.7, 19800, + "https://images.unsplash.com/photo-1639762681485-074b7f938ba0?w=800&h=400&fit=crop"), + + # Developer Tools + ("Crawl4AI VSCode", "Developer Tools", "Open Source", True, False, + "VSCode extension for Crawl4AI development", "https://marketplace.visualstudio.com", + "https://github.com/crawl4ai/vscode", "Free", 4.8, 34500, + "https://images.unsplash.com/photo-1629654297299-c8506221ca97?w=800&h=400&fit=crop"), + + ("Postman Collection", "Developer Tools", "Open Source", False, False, + "Postman collection for Crawl4AI API testing", "https://postman.com/crawl4ai", + "https://github.com/crawl4ai/postman", "Free", 4.3, 7800, + "https://images.unsplash.com/photo-1599507593499-a3f7d7d97667?w=800&h=400&fit=crop"), + + ("Debug Toolkit", "Developer Tools", "Open Source", False, False, + "Debugging tools for crawler development", "https://debug.crawl4ai.com", + "https://github.com/crawl4ai/debug", "Free", 4.0, 4300, + "https://images.unsplash.com/photo-1515879218367-8466d910aaa4?w=800&h=400&fit=crop"), + ] + + for name, category, type_, featured, sponsored, desc, url, github, pricing, rating, downloads, image in apps_data: + screenshots = json.dumps([ + f"https://images.unsplash.com/photo-{random.randint(1500000000000, 1700000000000)}-{random.randint(1000000000000, 9999999999999)}?w=800&h=600&fit=crop", + f"https://images.unsplash.com/photo-{random.randint(1500000000000, 1700000000000)}-{random.randint(1000000000000, 9999999999999)}?w=800&h=600&fit=crop" + ]) + cursor.execute(""" + INSERT INTO apps (name, slug, description, category, type, featured, sponsored, + website_url, github_url, pricing, rating, downloads, image, screenshots, logo_url, + integration_guide, contact_email, views) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, (name, generate_slug(name), desc, category, type_, featured, sponsored, + url, github, pricing, rating, downloads, image, screenshots, + f"https://ui-avatars.com/api/?name={name}&background=50ffff&color=070708&size=128", + f"# {name} Integration\n\n```python\nfrom crawl4ai import AsyncWebCrawler\n# Integration code coming soon...\n```", + f"contact@{generate_slug(name)}.com", + random.randint(100, 5000))) + + # Articles with real images + articles_data = [ + ("Browser Automation Showdown: Playwright vs Puppeteer vs Selenium", + "Review", "John Doe", ["Playwright Cloud", "Puppeteer Extra"], + ["browser-automation", "comparison", "2024"], + "https://images.unsplash.com/photo-1587620962725-abab7fe55159?w=1200&h=630&fit=crop"), + + ("Top 5 Proxy Services for Web Scraping in 2024", + "Comparison", "Jane Smith", ["BrightData", "SmartProxy", "ProxyMesh"], + ["proxy", "web-scraping", "guide"], + "https://images.unsplash.com/photo-1558494949-ef010cbdcc31?w=1200&h=630&fit=crop"), + + ("Integrating LLMs with Crawl4AI: A Complete Guide", + "Tutorial", "Crawl4AI Team", ["LangChain Crawl", "GPT Scraper", "Claude Extract"], + ["llm", "integration", "tutorial"], + "https://images.unsplash.com/photo-1677442136019-21780ecad995?w=1200&h=630&fit=crop"), + + ("Building Scalable Crawlers with Cloud Infrastructure", + "Tutorial", "Mike Johnson", ["BrowserCloud", "Browserless"], + ["cloud", "scalability", "architecture"], + "https://images.unsplash.com/photo-1667372393119-3d4c48d07fc9?w=1200&h=630&fit=crop"), + + ("What's New in Crawl4AI Marketplace", + "News", "Crawl4AI Team", [], + ["marketplace", "announcement", "news"], + "https://images.unsplash.com/photo-1556075798-4825dfaaf498?w=1200&h=630&fit=crop"), + + ("Cost Analysis: Self-Hosted vs Cloud Browser Solutions", + "Comparison", "Sarah Chen", ["BrowserCloud", "LambdaTest", "Browserless"], + ["cost", "cloud", "comparison"], + "https://images.unsplash.com/photo-1554224155-8d04cb21cd6c?w=1200&h=630&fit=crop"), + + ("Getting Started with Browser Automation", + "Tutorial", "Crawl4AI Team", ["Playwright Cloud", "Selenium Grid Hub"], + ["beginner", "tutorial", "automation"], + "https://images.unsplash.com/photo-1498050108023-c5249f4df085?w=1200&h=630&fit=crop"), + + ("The Future of Web Scraping: AI-Powered Extraction", + "News", "Dr. Alan Turing", ["Claude Extract", "GPT Scraper"], + ["ai", "future", "trends"], + "https://images.unsplash.com/photo-1593720213428-28a5b9e94613?w=1200&h=630&fit=crop") + ] + + for title, category, author, related_apps, tags, image in articles_data: + # Get app IDs for related apps + related_ids = [] + for app_name in related_apps: + cursor.execute("SELECT id FROM apps WHERE name = ?", (app_name,)) + result = cursor.fetchone() + if result: + related_ids.append(result[0]) + + content = f"""# {title} + +By {author} | {datetime.now().strftime('%B %d, %Y')} + +## Introduction + +This is a comprehensive article about {title.lower()}. Lorem ipsum dolor sit amet, consectetur adipiscing elit. +Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. + +## Key Points + +- Important point about the topic +- Another crucial insight +- Technical details and specifications +- Performance comparisons + +## Conclusion + +In summary, this article explored various aspects of the topic. Stay tuned for more updates! +""" + + cursor.execute(""" + INSERT INTO articles (title, slug, content, author, category, related_apps, + featured_image, tags, views) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, (title, generate_slug(title), content, author, category, + json.dumps(related_ids), image, json.dumps(tags), + random.randint(200, 10000))) + + # Sponsors + sponsors_data = [ + ("BrightData", "Gold", "https://brightdata.com", + "https://images.unsplash.com/photo-1558494949-ef010cbdcc31?w=728&h=90&fit=crop"), + ("ScraperAPI", "Gold", "https://scraperapi.com", + "https://images.unsplash.com/photo-1460925895917-afdab827c52f?w=728&h=90&fit=crop"), + ("BrowserCloud", "Silver", "https://browsercloud.io", + "https://images.unsplash.com/photo-1667372393119-3d4c48d07fc9?w=728&h=90&fit=crop"), + ("Claude Extract", "Silver", "https://claude-extract.com", + "https://images.unsplash.com/photo-1686191128892-3b09ad503b4f?w=728&h=90&fit=crop"), + ("SmartProxy", "Bronze", "https://smartproxy.com", + "https://images.unsplash.com/photo-1544197150-b99a580bb7a8?w=728&h=90&fit=crop") + ] + + for company, tier, landing_url, banner in sponsors_data: + start_date = datetime.now() - timedelta(days=random.randint(1, 30)) + end_date = datetime.now() + timedelta(days=random.randint(30, 180)) + + cursor.execute(""" + INSERT INTO sponsors (company_name, logo_url, tier, banner_url, + landing_url, active, start_date, end_date) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, (company, + f"https://ui-avatars.com/api/?name={company}&background=09b5a5&color=fff&size=200", + tier, banner, landing_url, 1, + start_date.isoformat(), end_date.isoformat())) + + conn.commit() + print("✓ Dummy data generated successfully!") + print(f" - {len(categories)} categories") + print(f" - {len(apps_data)} apps") + print(f" - {len(articles_data)} articles") + print(f" - {len(sponsors_data)} sponsors") + +if __name__ == "__main__": + generate_dummy_data() \ No newline at end of file diff --git a/docs/md_v2/marketplace/backend/requirements.txt b/docs/md_v2/marketplace/backend/requirements.txt new file mode 100644 index 00000000..1401b0e3 --- /dev/null +++ b/docs/md_v2/marketplace/backend/requirements.txt @@ -0,0 +1,5 @@ +fastapi +uvicorn +pyyaml +python-multipart +python-dotenv \ No newline at end of file diff --git a/docs/md_v2/marketplace/backend/schema.yaml b/docs/md_v2/marketplace/backend/schema.yaml new file mode 100644 index 00000000..c5f443d0 --- /dev/null +++ b/docs/md_v2/marketplace/backend/schema.yaml @@ -0,0 +1,75 @@ +database: + name: marketplace.db + +tables: + apps: + columns: + id: {type: INTEGER, primary: true, autoincrement: true} + name: {type: TEXT, required: true} + slug: {type: TEXT, unique: true} + description: {type: TEXT} + long_description: {type: TEXT} + logo_url: {type: TEXT} + image: {type: TEXT} + screenshots: {type: JSON, default: '[]'} + category: {type: TEXT} + type: {type: TEXT, default: 'Open Source'} + status: {type: TEXT, default: 'Active'} + website_url: {type: TEXT} + github_url: {type: TEXT} + demo_url: {type: TEXT} + video_url: {type: TEXT} + documentation_url: {type: TEXT} + support_url: {type: TEXT} + discord_url: {type: TEXT} + pricing: {type: TEXT} + rating: {type: REAL, default: 0.0} + downloads: {type: INTEGER, default: 0} + featured: {type: BOOLEAN, default: 0} + sponsored: {type: BOOLEAN, default: 0} + integration_guide: {type: TEXT} + documentation: {type: TEXT} + examples: {type: TEXT} + installation_command: {type: TEXT} + requirements: {type: TEXT} + changelog: {type: TEXT} + tags: {type: JSON, default: '[]'} + added_date: {type: DATETIME, default: CURRENT_TIMESTAMP} + updated_date: {type: DATETIME, default: CURRENT_TIMESTAMP} + contact_email: {type: TEXT} + views: {type: INTEGER, default: 0} + + articles: + columns: + id: {type: INTEGER, primary: true, autoincrement: true} + title: {type: TEXT, required: true} + slug: {type: TEXT, unique: true} + content: {type: TEXT} + author: {type: TEXT, default: 'Crawl4AI Team'} + category: {type: TEXT} + related_apps: {type: JSON, default: '[]'} + featured_image: {type: TEXT} + published_date: {type: DATETIME, default: CURRENT_TIMESTAMP} + tags: {type: JSON, default: '[]'} + views: {type: INTEGER, default: 0} + + categories: + columns: + id: {type: INTEGER, primary: true, autoincrement: true} + name: {type: TEXT, unique: true} + slug: {type: TEXT, unique: true} + icon: {type: TEXT} + description: {type: TEXT} + order_index: {type: INTEGER, default: 0} + + sponsors: + columns: + id: {type: INTEGER, primary: true, autoincrement: true} + company_name: {type: TEXT, required: true} + logo_url: {type: TEXT} + tier: {type: TEXT, default: 'Bronze'} + banner_url: {type: TEXT} + landing_url: {type: TEXT} + active: {type: BOOLEAN, default: 1} + start_date: {type: DATETIME} + end_date: {type: DATETIME} \ No newline at end of file diff --git a/docs/md_v2/marketplace/backend/server.py b/docs/md_v2/marketplace/backend/server.py new file mode 100644 index 00000000..37df188a --- /dev/null +++ b/docs/md_v2/marketplace/backend/server.py @@ -0,0 +1,390 @@ +from fastapi import FastAPI, HTTPException, Query, Depends, Body +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +from typing import Optional, List, Dict, Any +import json +import hashlib +import secrets +from database import DatabaseManager +from datetime import datetime, timedelta + +# Import configuration (will exit if .env not found or invalid) +from config import Config + +app = FastAPI(title="Crawl4AI Marketplace API") + +# Security setup +security = HTTPBearer() +tokens = {} # In production, use Redis or database for token storage + +# CORS configuration +app.add_middleware( + CORSMiddleware, + allow_origins=Config.ALLOWED_ORIGINS, + allow_credentials=True, + allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"], + allow_headers=["*"], + max_age=3600 +) + +# Initialize database with configurable path +db = DatabaseManager(Config.DATABASE_PATH) + +def json_response(data, cache_time=3600): + """Helper to return JSON with cache headers""" + return JSONResponse( + content=data, + headers={ + "Cache-Control": f"public, max-age={cache_time}", + "X-Content-Type-Options": "nosniff" + } + ) + +# ============= PUBLIC ENDPOINTS ============= + +@app.get("/api/apps") +async def get_apps( + category: Optional[str] = None, + type: Optional[str] = None, + featured: Optional[bool] = None, + sponsored: Optional[bool] = None, + limit: int = Query(default=20, le=10000), + offset: int = Query(default=0) +): + """Get apps with optional filters""" + where_clauses = [] + if category: + where_clauses.append(f"category = '{category}'") + if type: + where_clauses.append(f"type = '{type}'") + if featured is not None: + where_clauses.append(f"featured = {1 if featured else 0}") + if sponsored is not None: + where_clauses.append(f"sponsored = {1 if sponsored else 0}") + + where = " AND ".join(where_clauses) if where_clauses else None + apps = db.get_all('apps', limit=limit, offset=offset, where=where) + + # Parse JSON fields + for app in apps: + if app.get('screenshots'): + app['screenshots'] = json.loads(app['screenshots']) + + return json_response(apps) + +@app.get("/api/apps/{slug}") +async def get_app(slug: str): + """Get single app by slug""" + apps = db.get_all('apps', where=f"slug = '{slug}'", limit=1) + if not apps: + raise HTTPException(status_code=404, detail="App not found") + + app = apps[0] + if app.get('screenshots'): + app['screenshots'] = json.loads(app['screenshots']) + + return json_response(app) + +@app.get("/api/articles") +async def get_articles( + category: Optional[str] = None, + limit: int = Query(default=20, le=10000), + offset: int = Query(default=0) +): + """Get articles with optional category filter""" + where = f"category = '{category}'" if category else None + articles = db.get_all('articles', limit=limit, offset=offset, where=where) + + # Parse JSON fields + for article in articles: + if article.get('related_apps'): + article['related_apps'] = json.loads(article['related_apps']) + if article.get('tags'): + article['tags'] = json.loads(article['tags']) + + return json_response(articles) + +@app.get("/api/articles/{slug}") +async def get_article(slug: str): + """Get single article by slug""" + articles = db.get_all('articles', where=f"slug = '{slug}'", limit=1) + if not articles: + raise HTTPException(status_code=404, detail="Article not found") + + article = articles[0] + if article.get('related_apps'): + article['related_apps'] = json.loads(article['related_apps']) + if article.get('tags'): + article['tags'] = json.loads(article['tags']) + + return json_response(article) + +@app.get("/api/categories") +async def get_categories(): + """Get all categories ordered by index""" + categories = db.get_all('categories', limit=50) + categories.sort(key=lambda x: x.get('order_index', 0)) + return json_response(categories, cache_time=7200) + +@app.get("/api/sponsors") +async def get_sponsors(active: Optional[bool] = True): + """Get sponsors, default active only""" + where = f"active = {1 if active else 0}" if active is not None else None + sponsors = db.get_all('sponsors', where=where, limit=20) + + # Filter by date if active + if active: + now = datetime.now().isoformat() + sponsors = [s for s in sponsors + if (not s.get('start_date') or s['start_date'] <= now) and + (not s.get('end_date') or s['end_date'] >= now)] + + return json_response(sponsors) + +@app.get("/api/search") +async def search(q: str = Query(min_length=2)): + """Search across apps and articles""" + if len(q) < 2: + return json_response({}) + + results = db.search(q, tables=['apps', 'articles']) + + # Parse JSON fields in results + for table, items in results.items(): + for item in items: + if table == 'apps' and item.get('screenshots'): + item['screenshots'] = json.loads(item['screenshots']) + elif table == 'articles': + if item.get('related_apps'): + item['related_apps'] = json.loads(item['related_apps']) + if item.get('tags'): + item['tags'] = json.loads(item['tags']) + + return json_response(results, cache_time=1800) + +@app.get("/api/stats") +async def get_stats(): + """Get marketplace statistics""" + stats = { + "total_apps": len(db.get_all('apps', limit=10000)), + "total_articles": len(db.get_all('articles', limit=10000)), + "total_categories": len(db.get_all('categories', limit=1000)), + "active_sponsors": len(db.get_all('sponsors', where="active = 1", limit=1000)) + } + return json_response(stats, cache_time=1800) + +# ============= ADMIN AUTHENTICATION ============= + +def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)): + """Verify admin authentication token""" + token = credentials.credentials + if token not in tokens or tokens[token] < datetime.now(): + raise HTTPException(status_code=401, detail="Invalid or expired token") + return token + +@app.post("/api/admin/login") +async def admin_login(password: str = Body(..., embed=True)): + """Admin login with password""" + provided_hash = hashlib.sha256(password.encode()).hexdigest() + + if provided_hash != Config.ADMIN_PASSWORD_HASH: + # Log failed attempt in production + print(f"Failed login attempt at {datetime.now()}") + raise HTTPException(status_code=401, detail="Invalid password") + + # Generate secure token + token = secrets.token_urlsafe(32) + tokens[token] = datetime.now() + timedelta(hours=Config.TOKEN_EXPIRY_HOURS) + + return { + "token": token, + "expires_in": Config.TOKEN_EXPIRY_HOURS * 3600 + } + +# ============= ADMIN ENDPOINTS ============= + +@app.get("/api/admin/stats", dependencies=[Depends(verify_token)]) +async def get_admin_stats(): + """Get detailed admin statistics""" + stats = { + "apps": { + "total": len(db.get_all('apps', limit=10000)), + "featured": len(db.get_all('apps', where="featured = 1", limit=10000)), + "sponsored": len(db.get_all('apps', where="sponsored = 1", limit=10000)) + }, + "articles": len(db.get_all('articles', limit=10000)), + "categories": len(db.get_all('categories', limit=1000)), + "sponsors": { + "active": len(db.get_all('sponsors', where="active = 1", limit=1000)), + "total": len(db.get_all('sponsors', limit=10000)) + }, + "total_views": sum(app.get('views', 0) for app in db.get_all('apps', limit=10000)) + } + return stats + +# Apps CRUD +@app.post("/api/admin/apps", dependencies=[Depends(verify_token)]) +async def create_app(app_data: Dict[str, Any]): + """Create new app""" + try: + # Handle JSON fields + for field in ['screenshots', 'tags']: + if field in app_data and isinstance(app_data[field], list): + app_data[field] = json.dumps(app_data[field]) + + cursor = db.conn.cursor() + columns = ', '.join(app_data.keys()) + placeholders = ', '.join(['?' for _ in app_data]) + cursor.execute(f"INSERT INTO apps ({columns}) VALUES ({placeholders})", + list(app_data.values())) + db.conn.commit() + return {"id": cursor.lastrowid, "message": "App created"} + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + +@app.put("/api/admin/apps/{app_id}", dependencies=[Depends(verify_token)]) +async def update_app(app_id: int, app_data: Dict[str, Any]): + """Update app""" + try: + # Handle JSON fields + for field in ['screenshots', 'tags']: + if field in app_data and isinstance(app_data[field], list): + app_data[field] = json.dumps(app_data[field]) + + set_clause = ', '.join([f"{k} = ?" for k in app_data.keys()]) + cursor = db.conn.cursor() + cursor.execute(f"UPDATE apps SET {set_clause} WHERE id = ?", + list(app_data.values()) + [app_id]) + db.conn.commit() + return {"message": "App updated"} + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + +@app.delete("/api/admin/apps/{app_id}", dependencies=[Depends(verify_token)]) +async def delete_app(app_id: int): + """Delete app""" + cursor = db.conn.cursor() + cursor.execute("DELETE FROM apps WHERE id = ?", (app_id,)) + db.conn.commit() + return {"message": "App deleted"} + +# Articles CRUD +@app.post("/api/admin/articles", dependencies=[Depends(verify_token)]) +async def create_article(article_data: Dict[str, Any]): + """Create new article""" + try: + for field in ['related_apps', 'tags']: + if field in article_data and isinstance(article_data[field], list): + article_data[field] = json.dumps(article_data[field]) + + cursor = db.conn.cursor() + columns = ', '.join(article_data.keys()) + placeholders = ', '.join(['?' for _ in article_data]) + cursor.execute(f"INSERT INTO articles ({columns}) VALUES ({placeholders})", + list(article_data.values())) + db.conn.commit() + return {"id": cursor.lastrowid, "message": "Article created"} + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + +@app.put("/api/admin/articles/{article_id}", dependencies=[Depends(verify_token)]) +async def update_article(article_id: int, article_data: Dict[str, Any]): + """Update article""" + try: + for field in ['related_apps', 'tags']: + if field in article_data and isinstance(article_data[field], list): + article_data[field] = json.dumps(article_data[field]) + + set_clause = ', '.join([f"{k} = ?" for k in article_data.keys()]) + cursor = db.conn.cursor() + cursor.execute(f"UPDATE articles SET {set_clause} WHERE id = ?", + list(article_data.values()) + [article_id]) + db.conn.commit() + return {"message": "Article updated"} + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + +@app.delete("/api/admin/articles/{article_id}", dependencies=[Depends(verify_token)]) +async def delete_article(article_id: int): + """Delete article""" + cursor = db.conn.cursor() + cursor.execute("DELETE FROM articles WHERE id = ?", (article_id,)) + db.conn.commit() + return {"message": "Article deleted"} + +# Categories CRUD +@app.post("/api/admin/categories", dependencies=[Depends(verify_token)]) +async def create_category(category_data: Dict[str, Any]): + """Create new category""" + try: + cursor = db.conn.cursor() + columns = ', '.join(category_data.keys()) + placeholders = ', '.join(['?' for _ in category_data]) + cursor.execute(f"INSERT INTO categories ({columns}) VALUES ({placeholders})", + list(category_data.values())) + db.conn.commit() + return {"id": cursor.lastrowid, "message": "Category created"} + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + +@app.put("/api/admin/categories/{cat_id}", dependencies=[Depends(verify_token)]) +async def update_category(cat_id: int, category_data: Dict[str, Any]): + """Update category""" + try: + set_clause = ', '.join([f"{k} = ?" for k in category_data.keys()]) + cursor = db.conn.cursor() + cursor.execute(f"UPDATE categories SET {set_clause} WHERE id = ?", + list(category_data.values()) + [cat_id]) + db.conn.commit() + return {"message": "Category updated"} + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + +# Sponsors CRUD +@app.post("/api/admin/sponsors", dependencies=[Depends(verify_token)]) +async def create_sponsor(sponsor_data: Dict[str, Any]): + """Create new sponsor""" + try: + cursor = db.conn.cursor() + columns = ', '.join(sponsor_data.keys()) + placeholders = ', '.join(['?' for _ in sponsor_data]) + cursor.execute(f"INSERT INTO sponsors ({columns}) VALUES ({placeholders})", + list(sponsor_data.values())) + db.conn.commit() + return {"id": cursor.lastrowid, "message": "Sponsor created"} + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + +@app.put("/api/admin/sponsors/{sponsor_id}", dependencies=[Depends(verify_token)]) +async def update_sponsor(sponsor_id: int, sponsor_data: Dict[str, Any]): + """Update sponsor""" + try: + set_clause = ', '.join([f"{k} = ?" for k in sponsor_data.keys()]) + cursor = db.conn.cursor() + cursor.execute(f"UPDATE sponsors SET {set_clause} WHERE id = ?", + list(sponsor_data.values()) + [sponsor_id]) + db.conn.commit() + return {"message": "Sponsor updated"} + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + +@app.get("/") +async def root(): + """API info""" + return { + "name": "Crawl4AI Marketplace API", + "version": "1.0.0", + "endpoints": [ + "/api/apps", + "/api/articles", + "/api/categories", + "/api/sponsors", + "/api/search?q=query", + "/api/stats" + ] + } + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="127.0.0.1", port=8100) \ No newline at end of file diff --git a/docs/md_v2/marketplace/frontend/app-detail.css b/docs/md_v2/marketplace/frontend/app-detail.css new file mode 100644 index 00000000..9f04c13a --- /dev/null +++ b/docs/md_v2/marketplace/frontend/app-detail.css @@ -0,0 +1,462 @@ +/* App Detail Page Styles */ + +.app-detail-container { + min-height: 100vh; + background: var(--bg-dark); +} + +/* Back Button */ +.header-nav { + display: flex; + align-items: center; +} + +.back-btn { + padding: 0.5rem 1rem; + background: transparent; + border: 1px solid var(--border-color); + color: var(--primary-cyan); + text-decoration: none; + transition: all 0.2s; + font-size: 0.875rem; +} + +.back-btn:hover { + border-color: var(--primary-cyan); + background: rgba(80, 255, 255, 0.1); +} + +/* App Hero Section */ +.app-hero { + max-width: 1800px; + margin: 2rem auto; + padding: 0 2rem; +} + +.app-hero-content { + display: grid; + grid-template-columns: 1fr 2fr; + gap: 3rem; + background: linear-gradient(135deg, #1a1a2e, #0f0f1e); + border: 2px solid var(--primary-cyan); + padding: 2rem; + box-shadow: 0 0 30px rgba(80, 255, 255, 0.15), + inset 0 0 20px rgba(80, 255, 255, 0.05); +} + +.app-hero-image { + width: 100%; + height: 300px; + background: linear-gradient(135deg, rgba(80, 255, 255, 0.1), rgba(243, 128, 245, 0.05)); + background-size: cover; + background-position: center; + border: 1px solid var(--border-color); + display: flex; + align-items: center; + justify-content: center; + font-size: 4rem; + color: var(--primary-cyan); +} + +.app-badges { + display: flex; + gap: 0.5rem; + margin-bottom: 1rem; +} + +.app-badge { + padding: 0.3rem 0.6rem; + background: var(--bg-tertiary); + color: var(--text-secondary); + font-size: 0.75rem; + text-transform: uppercase; + font-weight: 600; +} + +.app-badge.featured { + background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal)); + color: var(--bg-dark); + box-shadow: 0 2px 10px rgba(80, 255, 255, 0.3); +} + +.app-badge.sponsored { + background: linear-gradient(135deg, var(--warning), #ff8c00); + color: var(--bg-dark); + box-shadow: 0 2px 10px rgba(245, 158, 11, 0.3); +} + +.app-hero-info h1 { + font-size: 2.5rem; + color: var(--primary-cyan); + margin: 0.5rem 0; + text-shadow: 0 0 20px rgba(80, 255, 255, 0.5); +} + +.app-tagline { + font-size: 1.1rem; + color: var(--text-secondary); + margin-bottom: 2rem; +} + +/* Stats */ +.app-stats { + display: flex; + gap: 2rem; + margin: 2rem 0; + padding: 1rem 0; + border-top: 1px solid var(--border-color); + border-bottom: 1px solid var(--border-color); +} + +.stat { + display: flex; + flex-direction: column; + gap: 0.25rem; +} + +.stat-value { + font-size: 1.5rem; + color: var(--primary-cyan); + font-weight: 600; +} + +.stat-label { + font-size: 0.875rem; + color: var(--text-tertiary); +} + +/* Action Buttons */ +.app-actions { + display: flex; + gap: 1rem; + margin: 2rem 0; +} + +.action-btn { + padding: 0.75rem 1.5rem; + border: 1px solid var(--border-color); + background: transparent; + color: var(--text-primary); + text-decoration: none; + display: inline-flex; + align-items: center; + gap: 0.5rem; + transition: all 0.2s; + cursor: pointer; + font-family: inherit; + font-size: 0.9rem; +} + +.action-btn.primary { + background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal)); + color: var(--bg-dark); + border-color: var(--primary-cyan); + font-weight: 600; +} + +.action-btn.primary:hover { + box-shadow: 0 4px 15px rgba(80, 255, 255, 0.3); + transform: translateY(-2px); +} + +.action-btn.secondary { + border-color: var(--accent-pink); + color: var(--accent-pink); +} + +.action-btn.secondary:hover { + background: rgba(243, 128, 245, 0.1); + box-shadow: 0 4px 15px rgba(243, 128, 245, 0.2); +} + +.action-btn.ghost { + border-color: var(--border-color); + color: var(--text-secondary); +} + +.action-btn.ghost:hover { + border-color: var(--primary-cyan); + color: var(--primary-cyan); +} + +/* Pricing */ +.pricing-info { + display: flex; + align-items: center; + gap: 1rem; + font-size: 1.1rem; +} + +.pricing-label { + color: var(--text-tertiary); +} + +.pricing-value { + color: var(--warning); + font-weight: 600; +} + +/* Navigation Tabs */ +.app-nav { + max-width: 1800px; + margin: 2rem auto 0; + padding: 0 2rem; + display: flex; + gap: 1rem; + border-bottom: 2px solid var(--border-color); +} + +.nav-tab { + padding: 1rem 1.5rem; + background: transparent; + border: none; + border-bottom: 2px solid transparent; + color: var(--text-secondary); + cursor: pointer; + transition: all 0.2s; + font-family: inherit; + font-size: 0.9rem; + margin-bottom: -2px; +} + +.nav-tab:hover { + color: var(--primary-cyan); +} + +.nav-tab.active { + color: var(--primary-cyan); + border-bottom-color: var(--primary-cyan); +} + +/* Content Sections */ +.app-content { + max-width: 1800px; + margin: 2rem auto; + padding: 0 2rem; +} + +.tab-content { + display: none; +} + +.tab-content.active { + display: block; +} + +.docs-content { + max-width: 1200px; + padding: 2rem; + background: var(--bg-secondary); + border: 1px solid var(--border-color); +} + +.docs-content h2 { + font-size: 1.8rem; + color: var(--primary-cyan); + margin-bottom: 1rem; + padding-bottom: 0.5rem; + border-bottom: 1px solid var(--border-color); +} + +.docs-content h3 { + font-size: 1.3rem; + color: var(--text-primary); + margin: 2rem 0 1rem; +} + +.docs-content h4 { + font-size: 1.1rem; + color: var(--accent-pink); + margin: 1.5rem 0 0.5rem; +} + +.docs-content p { + color: var(--text-secondary); + line-height: 1.6; + margin-bottom: 1rem; +} + +.docs-content code { + background: var(--bg-tertiary); + padding: 0.2rem 0.4rem; + color: var(--primary-cyan); + font-family: 'Dank Mono', Monaco, monospace; + font-size: 0.9em; +} + +/* Code Blocks */ +.code-block { + background: var(--bg-dark); + border: 1px solid var(--border-color); + margin: 1rem 0; + overflow: hidden; +} + +.code-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 0.5rem 1rem; + background: var(--bg-tertiary); + border-bottom: 1px solid var(--border-color); +} + +.code-lang { + color: var(--primary-cyan); + font-size: 0.875rem; + text-transform: uppercase; +} + +.copy-btn { + padding: 0.25rem 0.5rem; + background: transparent; + border: 1px solid var(--border-color); + color: var(--text-secondary); + cursor: pointer; + font-size: 0.75rem; + transition: all 0.2s; +} + +.copy-btn:hover { + border-color: var(--primary-cyan); + color: var(--primary-cyan); +} + +.code-block pre { + margin: 0; + padding: 1rem; + overflow-x: auto; +} + +.code-block code { + background: transparent; + padding: 0; + color: var(--text-secondary); + font-size: 0.875rem; + line-height: 1.5; +} + +/* Feature Grid */ +.feature-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + gap: 1rem; + margin: 2rem 0; +} + +.feature-card { + background: var(--bg-tertiary); + border: 1px solid var(--border-color); + padding: 1.5rem; + transition: all 0.2s; +} + +.feature-card:hover { + border-color: var(--primary-cyan); + background: rgba(80, 255, 255, 0.05); +} + +.feature-card h4 { + margin-top: 0; +} + +/* Info Box */ +.info-box { + background: linear-gradient(135deg, rgba(80, 255, 255, 0.05), rgba(243, 128, 245, 0.03)); + border: 1px solid var(--primary-cyan); + border-left: 4px solid var(--primary-cyan); + padding: 1.5rem; + margin: 2rem 0; +} + +.info-box h4 { + margin-top: 0; + color: var(--primary-cyan); +} + +/* Support Grid */ +.support-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + gap: 1rem; + margin: 2rem 0; +} + +.support-card { + background: var(--bg-tertiary); + border: 1px solid var(--border-color); + padding: 1.5rem; + text-align: center; +} + +.support-card h3 { + color: var(--primary-cyan); + margin-bottom: 0.5rem; +} + +/* Related Apps */ +.related-apps { + max-width: 1800px; + margin: 4rem auto; + padding: 0 2rem; +} + +.related-apps h2 { + font-size: 1.5rem; + color: var(--text-primary); + margin-bottom: 1.5rem; +} + +.related-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); + gap: 1rem; +} + +.related-app-card { + background: var(--bg-secondary); + border: 1px solid var(--border-color); + padding: 1rem; + cursor: pointer; + transition: all 0.2s; +} + +.related-app-card:hover { + border-color: var(--primary-cyan); + transform: translateY(-2px); +} + +/* Responsive */ +@media (max-width: 1024px) { + .app-hero-content { + grid-template-columns: 1fr; + } + + .app-stats { + justify-content: space-around; + } +} + +@media (max-width: 768px) { + .app-hero-info h1 { + font-size: 2rem; + } + + .app-actions { + flex-direction: column; + } + + .app-nav { + overflow-x: auto; + gap: 0; + } + + .nav-tab { + white-space: nowrap; + } + + .feature-grid, + .support-grid { + grid-template-columns: 1fr; + } +} \ No newline at end of file diff --git a/docs/md_v2/marketplace/frontend/app-detail.html b/docs/md_v2/marketplace/frontend/app-detail.html new file mode 100644 index 00000000..92b5a6dd --- /dev/null +++ b/docs/md_v2/marketplace/frontend/app-detail.html @@ -0,0 +1,234 @@ + + + + + + App Details - Crawl4AI Marketplace + + + + +
+ +
+
+
+
+ +

+ [ + Marketplace + ] +

+
+
+
+ ← Back to Marketplace +
+
+
+ + +
+
+
+ +
+
+
+ Open Source + + +
+

App Name

+

App description goes here

+ +
+
+ ★★★★★ + Rating +
+
+ 0 + Downloads +
+
+ Category + Category +
+
+ +
+ + → Visit Website + + + ⚡ View on GitHub + + +
+ +
+ Pricing: + Free +
+
+
+
+ + + + + +
+ +
+
+

Quick Start

+

Get started with this integration in just a few steps.

+ +

Installation

+
+
+ bash + +
+ 
pip install crawl4ai
+
+ +

Basic Usage

+
+
+ python + +
+ 
from crawl4ai import AsyncWebCrawler
+
+async def main():
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            # Your configuration here
+        )
+        print(result.markdown)
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())
+
+ +

Advanced Configuration

+

Customize the crawler with these advanced options:

+ +
+
+

🚀 Performance

+

Optimize crawling speed with parallel processing and caching strategies.

+
+
+

🔒 Authentication

+

Handle login forms, cookies, and session management automatically.

+
+
+

🎯 Extraction

+

Use CSS selectors, XPath, or AI-powered content extraction.

+
+
+

🔄 Proxy Support

+

Rotate proxies and bypass rate limiting with built-in proxy management.

+
+
+ +

Integration Example

+
+
+ python + +
+ 
from crawl4ai import AsyncWebCrawler
+from crawl4ai.extraction_strategy import LLMExtractionStrategy
+
+async def extract_with_llm():
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            extraction_strategy=LLMExtractionStrategy(
+                provider="openai",
+                api_key="your-api-key",
+                instruction="Extract product information"
+            ),
+            bypass_cache=True
+        )
+        return result.extracted_content
+
+# Run the extraction
+data = await extract_with_llm()
+print(data)
+
+ +
+

💡 Pro Tip

+

Use the bypass_cache=True parameter when you need fresh data, or set cache_mode="write" to update the cache with new content.

+
+
+
+ + +
+
+

Documentation

+

Complete documentation and API reference.

+ +
+
+ + +
+
+

Examples

+

Real-world examples and use cases.

+ +
+
+ + +
+
+

Support

+
+
+

📧 Contact

+

contact@example.com

+
+
+

🐛 Report Issues

+

Found a bug? Report it on GitHub Issues.

+
+
+

💬 Community

+

Join our Discord for help and discussions.

+
+
+
+
+
+ + +
+

Related Apps

+ +
+
+ + + + \ No newline at end of file diff --git a/docs/md_v2/marketplace/frontend/app-detail.js b/docs/md_v2/marketplace/frontend/app-detail.js new file mode 100644 index 00000000..82422f14 --- /dev/null +++ b/docs/md_v2/marketplace/frontend/app-detail.js @@ -0,0 +1,324 @@ +// App Detail Page JavaScript +const API_BASE = 'http://localhost:8100/api'; + +class AppDetailPage { + constructor() { + this.appSlug = this.getAppSlugFromURL(); + this.appData = null; + this.init(); + } + + getAppSlugFromURL() { + const params = new URLSearchParams(window.location.search); + return params.get('app') || ''; + } + + async init() { + if (!this.appSlug) { + window.location.href = 'index.html'; + return; + } + + await this.loadAppDetails(); + this.setupEventListeners(); + await this.loadRelatedApps(); + } + + async loadAppDetails() { + try { + const response = await fetch(`${API_BASE}/apps/${this.appSlug}`); + if (!response.ok) throw new Error('App not found'); + + this.appData = await response.json(); + this.renderAppDetails(); + } catch (error) { + console.error('Error loading app details:', error); + // Fallback to loading all apps and finding the right one + try { + const response = await fetch(`${API_BASE}/apps`); + const apps = await response.json(); + this.appData = apps.find(app => app.slug === this.appSlug || app.name.toLowerCase().replace(/\s+/g, '-') === this.appSlug); + if (this.appData) { + this.renderAppDetails(); + } else { + window.location.href = 'index.html'; + } + } catch (err) { + console.error('Error loading apps:', err); + window.location.href = 'index.html'; + } + } + } + + renderAppDetails() { + if (!this.appData) return; + + // Update title + document.title = `${this.appData.name} - Crawl4AI Marketplace`; + + // Hero image + const appImage = document.getElementById('app-image'); + if (this.appData.image) { + appImage.style.backgroundImage = `url('${this.appData.image}')`; + appImage.innerHTML = ''; + } else { + appImage.innerHTML = `[${this.appData.category || 'APP'}]`; + } + + // Basic info + document.getElementById('app-name').textContent = this.appData.name; + document.getElementById('app-description').textContent = this.appData.description; + document.getElementById('app-type').textContent = this.appData.type || 'Open Source'; + document.getElementById('app-category').textContent = this.appData.category; + document.getElementById('app-pricing').textContent = this.appData.pricing || 'Free'; + + // Badges + if (this.appData.featured) { + document.getElementById('app-featured').style.display = 'inline-block'; + } + if (this.appData.sponsored) { + document.getElementById('app-sponsored').style.display = 'inline-block'; + } + + // Stats + const rating = this.appData.rating || 0; + const stars = '★'.repeat(Math.floor(rating)) + '☆'.repeat(5 - Math.floor(rating)); + document.getElementById('app-rating').textContent = stars + ` ${rating}/5`; + document.getElementById('app-downloads').textContent = this.formatNumber(this.appData.downloads || 0); + + // Action buttons + const websiteBtn = document.getElementById('app-website'); + const githubBtn = document.getElementById('app-github'); + + if (this.appData.website_url) { + websiteBtn.href = this.appData.website_url; + } else { + websiteBtn.style.display = 'none'; + } + + if (this.appData.github_url) { + githubBtn.href = this.appData.github_url; + } else { + githubBtn.style.display = 'none'; + } + + // Contact + document.getElementById('app-contact').textContent = this.appData.contact_email || 'Not available'; + + // Integration guide + this.renderIntegrationGuide(); + } + + renderIntegrationGuide() { + // Installation code + const installCode = document.getElementById('install-code'); + if (this.appData.type === 'Open Source' && this.appData.github_url) { + installCode.textContent = `# Clone from GitHub +git clone ${this.appData.github_url} + +# Install dependencies +pip install -r requirements.txt`; + } else if (this.appData.name.toLowerCase().includes('api')) { + installCode.textContent = `# Install via pip +pip install ${this.appData.slug} + +# Or install from source +pip install git+${this.appData.github_url || 'https://github.com/example/repo'}`; + } + + // Usage code - customize based on category + const usageCode = document.getElementById('usage-code'); + if (this.appData.category === 'Browser Automation') { + usageCode.textContent = `from crawl4ai import AsyncWebCrawler +from ${this.appData.slug.replace(/-/g, '_')} import ${this.appData.name.replace(/\s+/g, '')} + +async def main(): + # Initialize ${this.appData.name} + automation = ${this.appData.name.replace(/\s+/g, '')}() + + async with AsyncWebCrawler() as crawler: + result = await crawler.arun( + url="https://example.com", + browser_config=automation.config, + wait_for="css:body" + ) + print(result.markdown)`; + } else if (this.appData.category === 'Proxy Services') { + usageCode.textContent = `from crawl4ai import AsyncWebCrawler +import ${this.appData.slug.replace(/-/g, '_')} + +# Configure proxy +proxy_config = { + "server": "${this.appData.website_url || 'https://proxy.example.com'}", + "username": "your_username", + "password": "your_password" +} + +async with AsyncWebCrawler(proxy=proxy_config) as crawler: + result = await crawler.arun( + url="https://example.com", + bypass_cache=True + ) + print(result.status_code)`; + } else if (this.appData.category === 'LLM Integration') { + usageCode.textContent = `from crawl4ai import AsyncWebCrawler +from crawl4ai.extraction_strategy import LLMExtractionStrategy + +# Configure LLM extraction +strategy = LLMExtractionStrategy( + provider="${this.appData.name.toLowerCase().includes('gpt') ? 'openai' : 'anthropic'}", + api_key="your-api-key", + model="${this.appData.name.toLowerCase().includes('gpt') ? 'gpt-4' : 'claude-3'}", + instruction="Extract structured data" +) + +async with AsyncWebCrawler() as crawler: + result = await crawler.arun( + url="https://example.com", + extraction_strategy=strategy + ) + print(result.extracted_content)`; + } + + // Integration example + const integrationCode = document.getElementById('integration-code'); + integrationCode.textContent = this.appData.integration_guide || +`# Complete ${this.appData.name} Integration Example + +from crawl4ai import AsyncWebCrawler +from crawl4ai.extraction_strategy import JsonCssExtractionStrategy +import json + +async def crawl_with_${this.appData.slug.replace(/-/g, '_')}(): + """ + Complete example showing how to use ${this.appData.name} + with Crawl4AI for production web scraping + """ + + # Define extraction schema + schema = { + "name": "ProductList", + "baseSelector": "div.product", + "fields": [ + {"name": "title", "selector": "h2", "type": "text"}, + {"name": "price", "selector": ".price", "type": "text"}, + {"name": "image", "selector": "img", "type": "attribute", "attribute": "src"}, + {"name": "link", "selector": "a", "type": "attribute", "attribute": "href"} + ] + } + + # Initialize crawler with ${this.appData.name} + async with AsyncWebCrawler( + browser_type="chromium", + headless=True, + verbose=True + ) as crawler: + + # Crawl with extraction + result = await crawler.arun( + url="https://example.com/products", + extraction_strategy=JsonCssExtractionStrategy(schema), + cache_mode="bypass", + wait_for="css:.product", + screenshot=True + ) + + # Process results + if result.success: + products = json.loads(result.extracted_content) + print(f"Found {len(products)} products") + + for product in products[:5]: + print(f"- {product['title']}: {product['price']}") + + return products + +# Run the crawler +if __name__ == "__main__": + import asyncio + asyncio.run(crawl_with_${this.appData.slug.replace(/-/g, '_')}())`; + } + + formatNumber(num) { + if (num >= 1000000) { + return (num / 1000000).toFixed(1) + 'M'; + } else if (num >= 1000) { + return (num / 1000).toFixed(1) + 'K'; + } + return num.toString(); + } + + setupEventListeners() { + // Tab switching + const tabs = document.querySelectorAll('.nav-tab'); + tabs.forEach(tab => { + tab.addEventListener('click', () => { + // Update active tab + tabs.forEach(t => t.classList.remove('active')); + tab.classList.add('active'); + + // Show corresponding content + const tabName = tab.dataset.tab; + document.querySelectorAll('.tab-content').forEach(content => { + content.classList.remove('active'); + }); + document.getElementById(`${tabName}-tab`).classList.add('active'); + }); + }); + + // Copy integration code + document.getElementById('copy-integration').addEventListener('click', () => { + const code = document.getElementById('integration-code').textContent; + navigator.clipboard.writeText(code).then(() => { + const btn = document.getElementById('copy-integration'); + const originalText = btn.innerHTML; + btn.innerHTML = '✓ Copied!'; + setTimeout(() => { + btn.innerHTML = originalText; + }, 2000); + }); + }); + + // Copy code buttons + document.querySelectorAll('.copy-btn').forEach(btn => { + btn.addEventListener('click', (e) => { + const codeBlock = e.target.closest('.code-block'); + const code = codeBlock.querySelector('code').textContent; + navigator.clipboard.writeText(code).then(() => { + btn.textContent = 'Copied!'; + setTimeout(() => { + btn.textContent = 'Copy'; + }, 2000); + }); + }); + }); + } + + async loadRelatedApps() { + try { + const response = await fetch(`${API_BASE}/apps?category=${encodeURIComponent(this.appData.category)}&limit=4`); + const apps = await response.json(); + + const relatedApps = apps.filter(app => app.slug !== this.appSlug).slice(0, 3); + const grid = document.getElementById('related-apps-grid'); + + grid.innerHTML = relatedApps.map(app => ` +
+

${app.name}

+

${app.description.substring(0, 100)}...

+
+ ${app.type} + ★ ${app.rating}/5 +
+
+ `).join(''); + } catch (error) { + console.error('Error loading related apps:', error); + } + } +} + +// Initialize when DOM is loaded +document.addEventListener('DOMContentLoaded', () => { + new AppDetailPage(); +}); \ No newline at end of file diff --git a/docs/md_v2/marketplace/frontend/index.html b/docs/md_v2/marketplace/frontend/index.html new file mode 100644 index 00000000..d034638d --- /dev/null +++ b/docs/md_v2/marketplace/frontend/index.html @@ -0,0 +1,147 @@ + + + + + + Marketplace - Crawl4AI + + + +
+ +
+
+
+
+ +

+ [ + Marketplace + ] +

+
+

Tools, Integrations & Resources for Web Crawling

+
+
+ Apps: -- + Articles: -- + Downloads: -- +
+
+
+ + +
+
+ > + + / +
+
+ + +
+
+ + +
+ +
+ +
+ + +
+ +
+ + + + + +
+ +
+
+

> Latest Apps

+ +
+
+ +
+
+ + +
+
+

> Latest Articles

+
+
+ +
+
+ + +
+
+

# Trending

+
+ + +
+

+ Submit Your Tool

+

Share your integration

+ Submit → +
+
+
+ + +
+
+

> More Apps

+ +
+
+ +
+
+
+ + +
+
+
+

About Marketplace

+

Discover tools and integrations built by the Crawl4AI community.

+
+
+

Become a Sponsor

+

Reach developers building with Crawl4AI

+ +
+
+
+

[ Crawl4AI Marketplace · Updated -- ]

+
+
+
+ + + + \ No newline at end of file diff --git a/docs/md_v2/marketplace/frontend/marketplace.css b/docs/md_v2/marketplace/frontend/marketplace.css new file mode 100644 index 00000000..ad26c344 --- /dev/null +++ b/docs/md_v2/marketplace/frontend/marketplace.css @@ -0,0 +1,957 @@ +/* Marketplace CSS - Magazine Style Terminal Theme */ +@import url('../../assets/styles.css'); + +:root { + --primary-cyan: #50ffff; + --primary-teal: #09b5a5; + --accent-pink: #f380f5; + --bg-dark: #070708; + --bg-secondary: #1a1a1a; + --bg-tertiary: #3f3f44; + --text-primary: #e8e9ed; + --text-secondary: #d5cec0; + --text-tertiary: #a3abba; + --border-color: #3f3f44; + --success: #50ff50; + --error: #ff3c74; + --warning: #f59e0b; +} + +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +body { + font-family: 'Dank Mono', Monaco, monospace; + background: var(--bg-dark); + color: var(--text-primary); + line-height: 1.6; +} + +/* Global link styles */ +a { + color: var(--primary-cyan); + text-decoration: none; + transition: color 0.2s; +} + +a:hover { + color: var(--accent-pink); +} + +.marketplace-container { + min-height: 100vh; +} + +/* Header */ +.marketplace-header { + background: var(--bg-secondary); + border-bottom: 1px solid var(--border-color); + padding: 1.5rem 0; +} + +.header-content { + max-width: 1800px; + margin: 0 auto; + padding: 0 2rem; + display: flex; + justify-content: space-between; + align-items: center; +} + +.logo-title { + display: flex; + align-items: center; + gap: 1rem; +} + +.header-logo { + height: 40px; + width: auto; + filter: brightness(1.2); +} + +.marketplace-header h1 { + font-size: 1.5rem; + color: var(--primary-cyan); + margin: 0; +} + +.ascii-border { + color: var(--border-color); +} + +.tagline { + font-size: 0.875rem; + color: var(--text-tertiary); + margin-top: 0.25rem; +} + +.header-stats { + display: flex; + gap: 2rem; +} + +.stat-item { + font-size: 0.875rem; + color: var(--text-secondary); +} + +.stat-item span { + color: var(--primary-cyan); + font-weight: 600; +} + +/* Search and Filter Bar */ +.search-filter-bar { + max-width: 1800px; + margin: 1.5rem auto; + padding: 0 2rem; + display: flex; + gap: 1rem; + align-items: center; +} + +.search-box { + flex: 1; + max-width: 500px; + display: flex; + align-items: center; + background: var(--bg-secondary); + border: 1px solid var(--border-color); + padding: 0.75rem 1rem; + transition: border-color 0.2s; +} + +.search-box:focus-within { + border-color: var(--primary-cyan); +} + +.search-icon { + color: var(--text-tertiary); + margin-right: 1rem; +} + +#search-input { + flex: 1; + background: transparent; + border: none; + color: var(--text-primary); + font-family: inherit; + font-size: 0.9rem; + outline: none; +} + +.search-box kbd { + font-size: 0.75rem; + padding: 0.2rem 0.5rem; + background: var(--bg-tertiary); + border: 1px solid var(--border-color); + color: var(--text-tertiary); +} + +.category-filter { + display: flex; + gap: 0.5rem; + flex-wrap: wrap; +} + +.filter-btn { + background: transparent; + border: 1px solid var(--border-color); + color: var(--text-secondary); + padding: 0.5rem 1rem; + font-family: inherit; + font-size: 0.875rem; + cursor: pointer; + transition: all 0.2s; +} + +.filter-btn:hover { + border-color: var(--primary-cyan); + color: var(--primary-cyan); +} + +.filter-btn.active { + background: var(--primary-cyan); + color: var(--bg-dark); + border-color: var(--primary-cyan); +} + +/* Magazine Layout */ +.magazine-layout { + max-width: 1800px; + margin: 0 auto; + padding: 0 2rem 4rem; + display: grid; + grid-template-columns: 1fr; + gap: 2rem; +} + +/* Hero Featured Section */ +.hero-featured { + grid-column: 1 / -1; + position: relative; +} + +.hero-featured::before { + content: ''; + position: absolute; + top: -20px; + left: -20px; + right: -20px; + bottom: -20px; + background: radial-gradient(ellipse at center, rgba(80, 255, 255, 0.05), transparent 70%); + pointer-events: none; + z-index: -1; +} + +.featured-hero-card { + background: linear-gradient(135deg, #1a1a2e, #0f0f1e); + border: 2px solid var(--primary-cyan); + box-shadow: 0 0 30px rgba(80, 255, 255, 0.15), + inset 0 0 20px rgba(80, 255, 255, 0.05); + height: 380px; + position: relative; + overflow: hidden; + cursor: pointer; + transition: all 0.3s ease; + display: flex; + flex-direction: column; +} + +.featured-hero-card:hover { + border-color: var(--accent-pink); + box-shadow: 0 0 40px rgba(243, 128, 245, 0.2), + inset 0 0 30px rgba(243, 128, 245, 0.05); + transform: translateY(-2px); +} + +.hero-image { + width: 100%; + height: 240px; + background: linear-gradient(135deg, rgba(80, 255, 255, 0.1), rgba(243, 128, 245, 0.05)); + background-size: cover; + background-position: center; + display: flex; + align-items: center; + justify-content: center; + font-size: 3rem; + color: var(--primary-cyan); + flex-shrink: 0; + position: relative; + filter: brightness(1.1) contrast(1.1); +} + +.hero-image::after { + content: ''; + position: absolute; + bottom: 0; + left: 0; + right: 0; + height: 60%; + background: linear-gradient(to top, rgba(10, 10, 20, 0.95), transparent); +} + +.hero-content { + padding: 1.5rem; +} + +.hero-badge { + display: inline-block; + padding: 0.3rem 0.6rem; + background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal)); + color: var(--bg-dark); + font-size: 0.7rem; + text-transform: uppercase; + margin-bottom: 0.5rem; + font-weight: 600; + box-shadow: 0 2px 10px rgba(80, 255, 255, 0.3); +} + +.hero-title { + font-size: 1.6rem; + color: var(--primary-cyan); + margin: 0.5rem 0; + text-shadow: 0 0 20px rgba(80, 255, 255, 0.5); +} + +.hero-description { + color: var(--text-secondary); + line-height: 1.5; +} + +.hero-meta { + display: flex; + gap: 1.5rem; + margin-top: 1rem; + font-size: 0.875rem; +} + +.hero-meta span { + color: var(--text-tertiary); +} + +.hero-meta span:first-child { + color: var(--warning); +} + +/* Secondary Featured */ +.secondary-featured { + grid-column: 1 / -1; + height: 380px; + display: flex; + align-items: stretch; +} + +.featured-secondary-cards { + width: 100%; + display: flex; + flex-direction: column; + gap: 0.75rem; + justify-content: space-between; +} + +.secondary-card { + background: linear-gradient(135deg, rgba(80, 255, 255, 0.03), rgba(243, 128, 245, 0.02)); + border: 1px solid rgba(80, 255, 255, 0.3); + cursor: pointer; + transition: all 0.3s ease; + display: flex; + overflow: hidden; + height: calc((380px - 1.5rem) / 3); + flex: 1; + box-shadow: 0 2px 10px rgba(0, 0, 0, 0.3); +} + +.secondary-card:hover { + border-color: var(--accent-pink); + background: linear-gradient(135deg, rgba(243, 128, 245, 0.05), rgba(80, 255, 255, 0.03)); + box-shadow: 0 4px 15px rgba(243, 128, 245, 0.2); + transform: translateX(-3px); +} + +.secondary-image { + width: 120px; + background: linear-gradient(135deg, var(--bg-tertiary), var(--bg-secondary)); + background-size: cover; + background-position: center; + display: flex; + align-items: center; + justify-content: center; + font-size: 1.5rem; + color: var(--primary-cyan); + flex-shrink: 0; +} + +.secondary-content { + flex: 1; + padding: 1rem; + display: flex; + flex-direction: column; + justify-content: space-between; +} + +.secondary-title { + font-size: 1rem; + color: var(--text-primary); + margin-bottom: 0.25rem; +} + +.secondary-desc { + font-size: 0.75rem; + color: var(--text-secondary); + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; +} + +.secondary-meta { + font-size: 0.75rem; + color: var(--text-tertiary); +} + +.secondary-meta span:last-child { + color: var(--warning); +} + +/* Sponsored Section */ +.sponsored-section { + grid-column: 1 / -1; + background: var(--bg-secondary); + border: 1px solid var(--warning); + padding: 1rem; + position: relative; +} + +.section-label { + position: absolute; + top: -0.5rem; + left: 1rem; + background: var(--bg-secondary); + padding: 0 0.5rem; + color: var(--warning); + font-size: 0.65rem; + letter-spacing: 0.1em; +} + +.sponsored-cards { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + gap: 1rem; +} + +.sponsor-card { + padding: 1rem; + background: var(--bg-tertiary); + border: 1px solid var(--border-color); +} + +.sponsor-card h4 { + color: var(--accent-pink); + margin-bottom: 0.5rem; +} + +.sponsor-card p { + color: var(--text-secondary); + font-size: 0.85rem; + margin-bottom: 0.75rem; +} + +.sponsor-card a { + color: var(--primary-cyan); + text-decoration: none; + font-size: 0.85rem; +} + +.sponsor-card a:hover { + color: var(--accent-pink); +} + +/* Main Content Grid */ +.main-content { + grid-column: 1 / -1; + display: grid; + grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); + gap: 2rem; +} + +/* Column Headers */ +.column-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 1rem; + border-bottom: 1px solid var(--border-color); + padding-bottom: 0.5rem; +} + +.column-header h2 { + font-size: 1.1rem; + color: var(--text-primary); +} + +.mini-filter { + background: var(--bg-tertiary); + border: 1px solid var(--border-color); + color: var(--text-primary); + padding: 0.25rem 0.5rem; + font-family: inherit; + font-size: 0.75rem; +} + +.ascii-icon { + color: var(--primary-cyan); +} + +/* Apps Column */ +.apps-compact-grid { + display: flex; + flex-direction: column; + gap: 0.75rem; +} + +.app-compact { + background: var(--bg-secondary); + border: 1px solid var(--border-color); + border-left: 3px solid var(--border-color); + padding: 0.75rem; + cursor: pointer; + transition: all 0.2s; +} + +.app-compact:hover { + border-color: var(--primary-cyan); + border-left-color: var(--accent-pink); + transform: translateX(2px); +} + +.app-compact-header { + display: flex; + justify-content: space-between; + font-size: 0.75rem; + color: var(--text-tertiary); + margin-bottom: 0.25rem; +} + +.app-compact-header span:first-child { + color: var(--primary-cyan); +} + +.app-compact-header span:last-child { + color: var(--warning); +} + +.app-compact-title { + font-size: 0.9rem; + color: var(--text-primary); + margin-bottom: 0.25rem; +} + +.app-compact-desc { + font-size: 0.75rem; + color: var(--text-secondary); + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; +} + +/* Articles Column */ +.articles-compact-list { + display: flex; + flex-direction: column; + gap: 1rem; +} + +.article-compact { + border-left: 2px solid var(--border-color); + padding-left: 1rem; + cursor: pointer; + transition: all 0.2s; +} + +.article-compact:hover { + border-left-color: var(--primary-cyan); +} + +.article-meta { + font-size: 0.7rem; + color: var(--text-tertiary); + margin-bottom: 0.25rem; +} + +.article-meta span:first-child { + color: var(--accent-pink); +} + +.article-title { + font-size: 0.9rem; + color: var(--text-primary); + margin-bottom: 0.25rem; +} + +.article-author { + font-size: 0.75rem; + color: var(--text-secondary); +} + +/* Trending Column */ +.trending-items { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.trending-item { + display: flex; + align-items: center; + gap: 0.75rem; + padding: 0.5rem; + background: var(--bg-secondary); + border: 1px solid var(--border-color); + cursor: pointer; + transition: all 0.2s; +} + +.trending-item:hover { + border-color: var(--primary-cyan); +} + +.trending-rank { + font-size: 1.2rem; + color: var(--primary-cyan); + width: 2rem; + text-align: center; +} + +.trending-info { + flex: 1; +} + +.trending-name { + font-size: 0.85rem; + color: var(--text-primary); +} + +.trending-stats { + font-size: 0.7rem; + color: var(--text-tertiary); +} + +/* Submit Box */ +.submit-box { + margin-top: 1.5rem; + background: var(--bg-secondary); + border: 1px solid var(--primary-cyan); + padding: 1rem; + text-align: center; +} + +.submit-box h3 { + font-size: 1rem; + color: var(--primary-cyan); + margin-bottom: 0.5rem; +} + +.submit-box p { + font-size: 0.8rem; + color: var(--text-secondary); + margin-bottom: 0.75rem; +} + +.submit-btn { + display: inline-block; + padding: 0.5rem 1rem; + background: transparent; + border: 1px solid var(--primary-cyan); + color: var(--primary-cyan); + text-decoration: none; + transition: all 0.2s; +} + +.submit-btn:hover { + background: var(--primary-cyan); + color: var(--bg-dark); +} + +/* More Apps Section */ +.more-apps { + grid-column: 1 / -1; + margin-top: 2rem; +} + +.section-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 1rem; +} + +.more-apps-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); + gap: 1rem; +} + +.load-more-btn { + background: transparent; + border: 1px solid var(--border-color); + color: var(--text-secondary); + padding: 0.5rem 1.5rem; + font-family: inherit; + cursor: pointer; + transition: all 0.2s; +} + +.load-more-btn:hover { + border-color: var(--primary-cyan); + color: var(--primary-cyan); +} + +/* Footer */ +.marketplace-footer { + background: var(--bg-secondary); + border-top: 1px solid var(--border-color); + margin-top: 4rem; + padding: 2rem 0; +} + +.footer-content { + max-width: 1800px; + margin: 0 auto; + padding: 0 2rem; + display: grid; + grid-template-columns: 1fr 1fr; + gap: 2rem; +} + +.footer-section h3 { + font-size: 1rem; + margin-bottom: 0.5rem; + color: var(--primary-cyan); +} + +.footer-section p { + font-size: 0.875rem; + color: var(--text-secondary); + margin-bottom: 1rem; +} + +.sponsor-btn { + display: inline-block; + padding: 0.5rem 1rem; + background: transparent; + border: 1px solid var(--primary-cyan); + color: var(--primary-cyan); + text-decoration: none; + transition: all 0.2s; +} + +.sponsor-btn:hover { + background: var(--primary-cyan); + color: var(--bg-dark); +} + +.footer-bottom { + max-width: 1800px; + margin: 2rem auto 0; + padding: 1rem 2rem 0; + border-top: 1px solid var(--border-color); + font-size: 0.75rem; + color: var(--text-tertiary); +} + +/* Modal */ +.modal { + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: rgba(0, 0, 0, 0.8); + display: flex; + align-items: center; + justify-content: center; + z-index: 1000; +} + +.modal.hidden { + display: none; +} + +.modal-content { + background: var(--bg-secondary); + border: 1px solid var(--primary-cyan); + max-width: 800px; + width: 90%; + max-height: 80vh; + overflow-y: auto; + position: relative; +} + +.modal-close { + position: absolute; + top: 1rem; + right: 1rem; + background: transparent; + border: 1px solid var(--border-color); + color: var(--text-primary); + padding: 0.25rem 0.5rem; + cursor: pointer; + font-size: 1.2rem; +} + +.modal-close:hover { + border-color: var(--error); + color: var(--error); +} + +.app-detail { + padding: 2rem; +} + +.app-detail h2 { + font-size: 1.5rem; + margin-bottom: 1rem; + color: var(--primary-cyan); +} + +/* Loading */ +.loading { + text-align: center; + padding: 2rem; + color: var(--text-tertiary); +} + +.no-results { + text-align: center; + padding: 2rem; + color: var(--text-tertiary); +} + +/* Responsive - Tablet */ +@media (min-width: 768px) { + .magazine-layout { + grid-template-columns: repeat(2, 1fr); + } + + .hero-featured { + grid-column: 1 / -1; + } + + .secondary-featured { + grid-column: 1 / -1; + } + + .sponsored-section { + grid-column: 1 / -1; + } + + .main-content { + grid-column: 1 / -1; + grid-template-columns: repeat(2, 1fr); + } +} + +/* Responsive - Desktop */ +@media (min-width: 1024px) { + .magazine-layout { + grid-template-columns: repeat(3, 1fr); + } + + .hero-featured { + grid-column: 1 / 3; + grid-row: 1; + } + + .secondary-featured { + grid-column: 3 / 4; + grid-row: 1; + } + + .featured-secondary-cards { + flex-direction: column; + } + + .sponsored-section { + grid-column: 1 / -1; + } + + .main-content { + grid-column: 1 / -1; + grid-template-columns: repeat(3, 1fr); + } +} + +/* Responsive - Wide Desktop */ +@media (min-width: 1400px) { + .magazine-layout { + grid-template-columns: repeat(4, 1fr); + } + + .hero-featured { + grid-column: 1 / 3; + } + + .secondary-featured { + grid-column: 3 / 5; + grid-row: 1; + } + + .featured-secondary-cards { + grid-template-columns: repeat(2, 1fr); + } + + .main-content { + grid-template-columns: repeat(4, 1fr); + } + + .apps-column { + grid-column: span 2; + } + + .more-apps-grid { + grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); + } +} + +/* Responsive - Ultra Wide Desktop (for coders with wide monitors) */ +@media (min-width: 1800px) { + .magazine-layout { + grid-template-columns: repeat(5, 1fr); + } + + .hero-featured { + grid-column: 1 / 3; + } + + .secondary-featured { + grid-column: 3 / 6; + } + + .featured-secondary-cards { + grid-template-columns: repeat(3, 1fr); + } + + .sponsored-section { + grid-column: 1 / -1; + } + + .sponsored-cards { + grid-template-columns: repeat(5, 1fr); + } + + .main-content { + grid-template-columns: repeat(5, 1fr); + } + + .apps-column { + grid-column: span 2; + } + + .articles-column { + grid-column: span 2; + } + + .more-apps-grid { + grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); + } +} + +/* Responsive - Mobile */ +@media (max-width: 767px) { + .header-content { + flex-direction: column; + gap: 1rem; + } + + .search-filter-bar { + flex-direction: column; + align-items: stretch; + } + + .search-box { + max-width: none; + } + + .magazine-layout { + padding: 0 1rem 2rem; + } + + .footer-content { + grid-template-columns: 1fr; + } + + .secondary-card { + flex-direction: column; + } + + .secondary-image { + width: 100%; + height: 150px; + } +} \ No newline at end of file diff --git a/docs/md_v2/marketplace/frontend/marketplace.js b/docs/md_v2/marketplace/frontend/marketplace.js new file mode 100644 index 00000000..cdc22114 --- /dev/null +++ b/docs/md_v2/marketplace/frontend/marketplace.js @@ -0,0 +1,395 @@ +// Marketplace JS - Magazine Layout +const API_BASE = 'http://localhost:8100/api'; +const CACHE_TTL = 3600000; // 1 hour in ms + +class MarketplaceCache { + constructor() { + this.prefix = 'c4ai_market_'; + } + + get(key) { + const item = localStorage.getItem(this.prefix + key); + if (!item) return null; + + const data = JSON.parse(item); + if (Date.now() > data.expires) { + localStorage.removeItem(this.prefix + key); + return null; + } + return data.value; + } + + set(key, value, ttl = CACHE_TTL) { + const data = { + value: value, + expires: Date.now() + ttl + }; + localStorage.setItem(this.prefix + key, JSON.stringify(data)); + } + + clear() { + Object.keys(localStorage) + .filter(k => k.startsWith(this.prefix)) + .forEach(k => localStorage.removeItem(k)); + } +} + +class MarketplaceAPI { + constructor() { + this.cache = new MarketplaceCache(); + this.searchTimeout = null; + } + + async fetch(endpoint, useCache = true) { + const cacheKey = endpoint.replace(/[^\w]/g, '_'); + + if (useCache) { + const cached = this.cache.get(cacheKey); + if (cached) return cached; + } + + try { + const response = await fetch(`${API_BASE}${endpoint}`); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + + const data = await response.json(); + this.cache.set(cacheKey, data); + return data; + } catch (error) { + console.error('API Error:', error); + return null; + } + } + + async getStats() { + return this.fetch('/stats'); + } + + async getCategories() { + return this.fetch('/categories'); + } + + async getApps(params = {}) { + const query = new URLSearchParams(params).toString(); + return this.fetch(`/apps${query ? '?' + query : ''}`); + } + + async getArticles(params = {}) { + const query = new URLSearchParams(params).toString(); + return this.fetch(`/articles${query ? '?' + query : ''}`); + } + + async getSponsors() { + return this.fetch('/sponsors'); + } + + async search(query) { + if (query.length < 2) return {}; + return this.fetch(`/search?q=${encodeURIComponent(query)}`, false); + } +} + +class MarketplaceUI { + constructor() { + this.api = new MarketplaceAPI(); + this.currentCategory = 'all'; + this.currentType = ''; + this.searchTimeout = null; + this.loadedApps = 10; + this.init(); + } + + async init() { + await this.loadStats(); + await this.loadCategories(); + await this.loadFeaturedContent(); + await this.loadSponsors(); + await this.loadMainContent(); + this.setupEventListeners(); + } + + async loadStats() { + const stats = await this.api.getStats(); + if (stats) { + document.getElementById('total-apps').textContent = stats.total_apps || '0'; + document.getElementById('total-articles').textContent = stats.total_articles || '0'; + document.getElementById('total-downloads').textContent = stats.total_downloads || '0'; + document.getElementById('last-update').textContent = new Date().toLocaleDateString(); + } + } + + async loadCategories() { + const categories = await this.api.getCategories(); + if (!categories) return; + + const filter = document.getElementById('category-filter'); + categories.forEach(cat => { + const btn = document.createElement('button'); + btn.className = 'filter-btn'; + btn.dataset.category = cat.slug; + btn.textContent = cat.name; + btn.onclick = () => this.filterByCategory(cat.slug); + filter.appendChild(btn); + }); + } + + async loadFeaturedContent() { + // Load hero featured + const featured = await this.api.getApps({ featured: true, limit: 4 }); + if (!featured || !featured.length) return; + + // Hero card (first featured) + const hero = featured[0]; + const heroCard = document.getElementById('featured-hero'); + if (hero) { + const imageUrl = hero.image || ''; + heroCard.innerHTML = ` +
+ ${!imageUrl ? `[${hero.category || 'APP'}]` : ''} +
+
+ ${hero.type || 'PAID'} +

${hero.name}

+

${hero.description}

+
+ ★ ${hero.rating || 0}/5 + ${hero.downloads || 0} downloads +
+
+ `; + heroCard.onclick = () => this.showAppDetail(hero); + } + + // Secondary featured cards + const secondary = document.getElementById('featured-secondary'); + secondary.innerHTML = ''; + if (featured.length > 1) { + featured.slice(1, 4).forEach(app => { + const card = document.createElement('div'); + card.className = 'secondary-card'; + const imageUrl = app.image || ''; + card.innerHTML = ` +
+ ${!imageUrl ? `[${app.category || 'APP'}]` : ''} +
+
+

${app.name}

+

${(app.description || '').substring(0, 100)}...

+
+ ${app.type || 'Open Source'} · ★ ${app.rating || 0}/5 +
+
+ `; + card.onclick = () => this.showAppDetail(app); + secondary.appendChild(card); + }); + } + } + + async loadSponsors() { + const sponsors = await this.api.getSponsors(); + if (!sponsors || !sponsors.length) { + // Show placeholder if no sponsors + const container = document.getElementById('sponsored-content'); + container.innerHTML = ` + + `; + return; + } + + const container = document.getElementById('sponsored-content'); + container.innerHTML = sponsors.slice(0, 5).map(sponsor => ` + + `).join(''); + } + + async loadMainContent() { + // Load apps column + const apps = await this.api.getApps({ limit: 8 }); + if (apps && apps.length) { + const appsGrid = document.getElementById('apps-grid'); + appsGrid.innerHTML = apps.map(app => ` +
+
+ ${app.category} + ★ ${app.rating}/5 +
+
${app.name}
+
${app.description}
+
+ `).join(''); + } + + // Load articles column + const articles = await this.api.getArticles({ limit: 6 }); + if (articles && articles.length) { + const articlesList = document.getElementById('articles-list'); + articlesList.innerHTML = articles.map(article => ` +
+
+ ${article.category} · ${new Date(article.published_at).toLocaleDateString()} +
+
${article.title}
+
by ${article.author}
+
+ `).join(''); + } + + // Load trending + if (apps && apps.length) { + const trending = apps.slice(0, 5); + const trendingList = document.getElementById('trending-list'); + trendingList.innerHTML = trending.map((app, i) => ` +
+
${i + 1}
+
+
${app.name}
+
${app.downloads} downloads
+
+
+ `).join(''); + } + + // Load more apps grid + const moreApps = await this.api.getApps({ offset: 8, limit: 12 }); + if (moreApps && moreApps.length) { + const moreGrid = document.getElementById('more-apps-grid'); + moreGrid.innerHTML = moreApps.map(app => ` +
+
+ ${app.category} + ${app.type} +
+
${app.name}
+
+ `).join(''); + } + } + + setupEventListeners() { + // Search + const searchInput = document.getElementById('search-input'); + searchInput.addEventListener('input', (e) => { + clearTimeout(this.searchTimeout); + this.searchTimeout = setTimeout(() => this.search(e.target.value), 300); + }); + + // Keyboard shortcut + document.addEventListener('keydown', (e) => { + if (e.key === '/' && !searchInput.contains(document.activeElement)) { + e.preventDefault(); + searchInput.focus(); + } + if (e.key === 'Escape' && searchInput.contains(document.activeElement)) { + searchInput.blur(); + searchInput.value = ''; + } + }); + + // Type filter + const typeFilter = document.getElementById('type-filter'); + typeFilter.addEventListener('change', (e) => { + this.currentType = e.target.value; + this.loadMainContent(); + }); + + // Load more + const loadMore = document.getElementById('load-more'); + loadMore.addEventListener('click', () => this.loadMoreApps()); + } + + async filterByCategory(category) { + // Update active state + document.querySelectorAll('.filter-btn').forEach(btn => { + btn.classList.toggle('active', btn.dataset.category === category); + }); + + this.currentCategory = category; + await this.loadMainContent(); + } + + async search(query) { + if (!query) { + await this.loadMainContent(); + return; + } + + const results = await this.api.search(query); + if (!results) return; + + // Update apps grid with search results + if (results.apps && results.apps.length) { + const appsGrid = document.getElementById('apps-grid'); + appsGrid.innerHTML = results.apps.map(app => ` +
+
+ ${app.category} + ★ ${app.rating}/5 +
+
${app.name}
+
${app.description}
+
+ `).join(''); + } + + // Update articles with search results + if (results.articles && results.articles.length) { + const articlesList = document.getElementById('articles-list'); + articlesList.innerHTML = results.articles.map(article => ` +
+
+ ${article.category} · ${new Date(article.published_at).toLocaleDateString()} +
+
${article.title}
+
by ${article.author}
+
+ `).join(''); + } + } + + async loadMoreApps() { + this.loadedApps += 12; + const moreApps = await this.api.getApps({ offset: this.loadedApps, limit: 12 }); + if (moreApps && moreApps.length) { + const moreGrid = document.getElementById('more-apps-grid'); + moreApps.forEach(app => { + const card = document.createElement('div'); + card.className = 'app-compact'; + card.innerHTML = ` +
+ ${app.category} + ${app.type} +
+
${app.name}
+ `; + card.onclick = () => this.showAppDetail(app); + moreGrid.appendChild(card); + }); + } + } + + showAppDetail(app) { + // Navigate to detail page instead of showing modal + const slug = app.slug || app.name.toLowerCase().replace(/\s+/g, '-'); + window.location.href = `app-detail.html?app=${slug}`; + } + + showArticle(articleId) { + // Could create article detail page similarly + console.log('Show article:', articleId); + } +} + +// Initialize marketplace +let marketplace; +document.addEventListener('DOMContentLoaded', () => { + marketplace = new MarketplaceUI(); +}); \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 50f19fce..d39172f6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,6 +14,8 @@ nav: - "Demo Apps": "apps/index.md" - "C4A-Script Editor": "apps/c4a-script/index.html" - "LLM Context Builder": "apps/llmtxt/index.html" + - "Marketplace": "marketplace/frontend/index.html" + - "Marketplace Admin": "marketplace/admin/index.html" - Setup & Installation: - "Installation": "core/installation.md" - "Docker Deployment": "core/docker-deployment.md" From 749d200866b739595297f6a1aee0e39d67586806 Mon Sep 17 00:00:00 2001 From: unclecode Date: Thu, 2 Oct 2025 17:08:50 +0800 Subject: [PATCH 043/119] fix(marketplace): Update URLs to use /marketplace path and relative API endpoints - Change API_BASE to relative '/api' for production - Move marketplace to /marketplace instead of /marketplace/frontend - Update MkDocs navigation - Fix logo path in marketplace index --- docs/md_v2/marketplace/admin/admin.js | 2 +- docs/md_v2/marketplace/app-detail.css | 462 +++++++++ docs/md_v2/marketplace/app-detail.js | 324 ++++++ docs/md_v2/marketplace/frontend/app-detail.js | 2 +- .../md_v2/marketplace/frontend/marketplace.js | 2 +- docs/md_v2/marketplace/index.html | 147 +++ docs/md_v2/marketplace/marketplace.css | 957 ++++++++++++++++++ docs/md_v2/marketplace/marketplace.js | 395 ++++++++ mkdocs.yml | 2 +- 9 files changed, 2289 insertions(+), 4 deletions(-) create mode 100644 docs/md_v2/marketplace/app-detail.css create mode 100644 docs/md_v2/marketplace/app-detail.js create mode 100644 docs/md_v2/marketplace/index.html create mode 100644 docs/md_v2/marketplace/marketplace.css create mode 100644 docs/md_v2/marketplace/marketplace.js diff --git a/docs/md_v2/marketplace/admin/admin.js b/docs/md_v2/marketplace/admin/admin.js index 861d3ba7..258858da 100644 --- a/docs/md_v2/marketplace/admin/admin.js +++ b/docs/md_v2/marketplace/admin/admin.js @@ -1,5 +1,5 @@ // Admin Dashboard - Smart & Powerful -const API_BASE = 'http://localhost:8100/api'; +const API_BASE = '/api'; class AdminDashboard { constructor() { diff --git a/docs/md_v2/marketplace/app-detail.css b/docs/md_v2/marketplace/app-detail.css new file mode 100644 index 00000000..9f04c13a --- /dev/null +++ b/docs/md_v2/marketplace/app-detail.css @@ -0,0 +1,462 @@ +/* App Detail Page Styles */ + +.app-detail-container { + min-height: 100vh; + background: var(--bg-dark); +} + +/* Back Button */ +.header-nav { + display: flex; + align-items: center; +} + +.back-btn { + padding: 0.5rem 1rem; + background: transparent; + border: 1px solid var(--border-color); + color: var(--primary-cyan); + text-decoration: none; + transition: all 0.2s; + font-size: 0.875rem; +} + +.back-btn:hover { + border-color: var(--primary-cyan); + background: rgba(80, 255, 255, 0.1); +} + +/* App Hero Section */ +.app-hero { + max-width: 1800px; + margin: 2rem auto; + padding: 0 2rem; +} + +.app-hero-content { + display: grid; + grid-template-columns: 1fr 2fr; + gap: 3rem; + background: linear-gradient(135deg, #1a1a2e, #0f0f1e); + border: 2px solid var(--primary-cyan); + padding: 2rem; + box-shadow: 0 0 30px rgba(80, 255, 255, 0.15), + inset 0 0 20px rgba(80, 255, 255, 0.05); +} + +.app-hero-image { + width: 100%; + height: 300px; + background: linear-gradient(135deg, rgba(80, 255, 255, 0.1), rgba(243, 128, 245, 0.05)); + background-size: cover; + background-position: center; + border: 1px solid var(--border-color); + display: flex; + align-items: center; + justify-content: center; + font-size: 4rem; + color: var(--primary-cyan); +} + +.app-badges { + display: flex; + gap: 0.5rem; + margin-bottom: 1rem; +} + +.app-badge { + padding: 0.3rem 0.6rem; + background: var(--bg-tertiary); + color: var(--text-secondary); + font-size: 0.75rem; + text-transform: uppercase; + font-weight: 600; +} + +.app-badge.featured { + background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal)); + color: var(--bg-dark); + box-shadow: 0 2px 10px rgba(80, 255, 255, 0.3); +} + +.app-badge.sponsored { + background: linear-gradient(135deg, var(--warning), #ff8c00); + color: var(--bg-dark); + box-shadow: 0 2px 10px rgba(245, 158, 11, 0.3); +} + +.app-hero-info h1 { + font-size: 2.5rem; + color: var(--primary-cyan); + margin: 0.5rem 0; + text-shadow: 0 0 20px rgba(80, 255, 255, 0.5); +} + +.app-tagline { + font-size: 1.1rem; + color: var(--text-secondary); + margin-bottom: 2rem; +} + +/* Stats */ +.app-stats { + display: flex; + gap: 2rem; + margin: 2rem 0; + padding: 1rem 0; + border-top: 1px solid var(--border-color); + border-bottom: 1px solid var(--border-color); +} + +.stat { + display: flex; + flex-direction: column; + gap: 0.25rem; +} + +.stat-value { + font-size: 1.5rem; + color: var(--primary-cyan); + font-weight: 600; +} + +.stat-label { + font-size: 0.875rem; + color: var(--text-tertiary); +} + +/* Action Buttons */ +.app-actions { + display: flex; + gap: 1rem; + margin: 2rem 0; +} + +.action-btn { + padding: 0.75rem 1.5rem; + border: 1px solid var(--border-color); + background: transparent; + color: var(--text-primary); + text-decoration: none; + display: inline-flex; + align-items: center; + gap: 0.5rem; + transition: all 0.2s; + cursor: pointer; + font-family: inherit; + font-size: 0.9rem; +} + +.action-btn.primary { + background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal)); + color: var(--bg-dark); + border-color: var(--primary-cyan); + font-weight: 600; +} + +.action-btn.primary:hover { + box-shadow: 0 4px 15px rgba(80, 255, 255, 0.3); + transform: translateY(-2px); +} + +.action-btn.secondary { + border-color: var(--accent-pink); + color: var(--accent-pink); +} + +.action-btn.secondary:hover { + background: rgba(243, 128, 245, 0.1); + box-shadow: 0 4px 15px rgba(243, 128, 245, 0.2); +} + +.action-btn.ghost { + border-color: var(--border-color); + color: var(--text-secondary); +} + +.action-btn.ghost:hover { + border-color: var(--primary-cyan); + color: var(--primary-cyan); +} + +/* Pricing */ +.pricing-info { + display: flex; + align-items: center; + gap: 1rem; + font-size: 1.1rem; +} + +.pricing-label { + color: var(--text-tertiary); +} + +.pricing-value { + color: var(--warning); + font-weight: 600; +} + +/* Navigation Tabs */ +.app-nav { + max-width: 1800px; + margin: 2rem auto 0; + padding: 0 2rem; + display: flex; + gap: 1rem; + border-bottom: 2px solid var(--border-color); +} + +.nav-tab { + padding: 1rem 1.5rem; + background: transparent; + border: none; + border-bottom: 2px solid transparent; + color: var(--text-secondary); + cursor: pointer; + transition: all 0.2s; + font-family: inherit; + font-size: 0.9rem; + margin-bottom: -2px; +} + +.nav-tab:hover { + color: var(--primary-cyan); +} + +.nav-tab.active { + color: var(--primary-cyan); + border-bottom-color: var(--primary-cyan); +} + +/* Content Sections */ +.app-content { + max-width: 1800px; + margin: 2rem auto; + padding: 0 2rem; +} + +.tab-content { + display: none; +} + +.tab-content.active { + display: block; +} + +.docs-content { + max-width: 1200px; + padding: 2rem; + background: var(--bg-secondary); + border: 1px solid var(--border-color); +} + +.docs-content h2 { + font-size: 1.8rem; + color: var(--primary-cyan); + margin-bottom: 1rem; + padding-bottom: 0.5rem; + border-bottom: 1px solid var(--border-color); +} + +.docs-content h3 { + font-size: 1.3rem; + color: var(--text-primary); + margin: 2rem 0 1rem; +} + +.docs-content h4 { + font-size: 1.1rem; + color: var(--accent-pink); + margin: 1.5rem 0 0.5rem; +} + +.docs-content p { + color: var(--text-secondary); + line-height: 1.6; + margin-bottom: 1rem; +} + +.docs-content code { + background: var(--bg-tertiary); + padding: 0.2rem 0.4rem; + color: var(--primary-cyan); + font-family: 'Dank Mono', Monaco, monospace; + font-size: 0.9em; +} + +/* Code Blocks */ +.code-block { + background: var(--bg-dark); + border: 1px solid var(--border-color); + margin: 1rem 0; + overflow: hidden; +} + +.code-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 0.5rem 1rem; + background: var(--bg-tertiary); + border-bottom: 1px solid var(--border-color); +} + +.code-lang { + color: var(--primary-cyan); + font-size: 0.875rem; + text-transform: uppercase; +} + +.copy-btn { + padding: 0.25rem 0.5rem; + background: transparent; + border: 1px solid var(--border-color); + color: var(--text-secondary); + cursor: pointer; + font-size: 0.75rem; + transition: all 0.2s; +} + +.copy-btn:hover { + border-color: var(--primary-cyan); + color: var(--primary-cyan); +} + +.code-block pre { + margin: 0; + padding: 1rem; + overflow-x: auto; +} + +.code-block code { + background: transparent; + padding: 0; + color: var(--text-secondary); + font-size: 0.875rem; + line-height: 1.5; +} + +/* Feature Grid */ +.feature-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + gap: 1rem; + margin: 2rem 0; +} + +.feature-card { + background: var(--bg-tertiary); + border: 1px solid var(--border-color); + padding: 1.5rem; + transition: all 0.2s; +} + +.feature-card:hover { + border-color: var(--primary-cyan); + background: rgba(80, 255, 255, 0.05); +} + +.feature-card h4 { + margin-top: 0; +} + +/* Info Box */ +.info-box { + background: linear-gradient(135deg, rgba(80, 255, 255, 0.05), rgba(243, 128, 245, 0.03)); + border: 1px solid var(--primary-cyan); + border-left: 4px solid var(--primary-cyan); + padding: 1.5rem; + margin: 2rem 0; +} + +.info-box h4 { + margin-top: 0; + color: var(--primary-cyan); +} + +/* Support Grid */ +.support-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + gap: 1rem; + margin: 2rem 0; +} + +.support-card { + background: var(--bg-tertiary); + border: 1px solid var(--border-color); + padding: 1.5rem; + text-align: center; +} + +.support-card h3 { + color: var(--primary-cyan); + margin-bottom: 0.5rem; +} + +/* Related Apps */ +.related-apps { + max-width: 1800px; + margin: 4rem auto; + padding: 0 2rem; +} + +.related-apps h2 { + font-size: 1.5rem; + color: var(--text-primary); + margin-bottom: 1.5rem; +} + +.related-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); + gap: 1rem; +} + +.related-app-card { + background: var(--bg-secondary); + border: 1px solid var(--border-color); + padding: 1rem; + cursor: pointer; + transition: all 0.2s; +} + +.related-app-card:hover { + border-color: var(--primary-cyan); + transform: translateY(-2px); +} + +/* Responsive */ +@media (max-width: 1024px) { + .app-hero-content { + grid-template-columns: 1fr; + } + + .app-stats { + justify-content: space-around; + } +} + +@media (max-width: 768px) { + .app-hero-info h1 { + font-size: 2rem; + } + + .app-actions { + flex-direction: column; + } + + .app-nav { + overflow-x: auto; + gap: 0; + } + + .nav-tab { + white-space: nowrap; + } + + .feature-grid, + .support-grid { + grid-template-columns: 1fr; + } +} \ No newline at end of file diff --git a/docs/md_v2/marketplace/app-detail.js b/docs/md_v2/marketplace/app-detail.js new file mode 100644 index 00000000..d1b3b559 --- /dev/null +++ b/docs/md_v2/marketplace/app-detail.js @@ -0,0 +1,324 @@ +// App Detail Page JavaScript +const API_BASE = '/api'; + +class AppDetailPage { + constructor() { + this.appSlug = this.getAppSlugFromURL(); + this.appData = null; + this.init(); + } + + getAppSlugFromURL() { + const params = new URLSearchParams(window.location.search); + return params.get('app') || ''; + } + + async init() { + if (!this.appSlug) { + window.location.href = 'index.html'; + return; + } + + await this.loadAppDetails(); + this.setupEventListeners(); + await this.loadRelatedApps(); + } + + async loadAppDetails() { + try { + const response = await fetch(`${API_BASE}/apps/${this.appSlug}`); + if (!response.ok) throw new Error('App not found'); + + this.appData = await response.json(); + this.renderAppDetails(); + } catch (error) { + console.error('Error loading app details:', error); + // Fallback to loading all apps and finding the right one + try { + const response = await fetch(`${API_BASE}/apps`); + const apps = await response.json(); + this.appData = apps.find(app => app.slug === this.appSlug || app.name.toLowerCase().replace(/\s+/g, '-') === this.appSlug); + if (this.appData) { + this.renderAppDetails(); + } else { + window.location.href = 'index.html'; + } + } catch (err) { + console.error('Error loading apps:', err); + window.location.href = 'index.html'; + } + } + } + + renderAppDetails() { + if (!this.appData) return; + + // Update title + document.title = `${this.appData.name} - Crawl4AI Marketplace`; + + // Hero image + const appImage = document.getElementById('app-image'); + if (this.appData.image) { + appImage.style.backgroundImage = `url('${this.appData.image}')`; + appImage.innerHTML = ''; + } else { + appImage.innerHTML = `[${this.appData.category || 'APP'}]`; + } + + // Basic info + document.getElementById('app-name').textContent = this.appData.name; + document.getElementById('app-description').textContent = this.appData.description; + document.getElementById('app-type').textContent = this.appData.type || 'Open Source'; + document.getElementById('app-category').textContent = this.appData.category; + document.getElementById('app-pricing').textContent = this.appData.pricing || 'Free'; + + // Badges + if (this.appData.featured) { + document.getElementById('app-featured').style.display = 'inline-block'; + } + if (this.appData.sponsored) { + document.getElementById('app-sponsored').style.display = 'inline-block'; + } + + // Stats + const rating = this.appData.rating || 0; + const stars = '★'.repeat(Math.floor(rating)) + '☆'.repeat(5 - Math.floor(rating)); + document.getElementById('app-rating').textContent = stars + ` ${rating}/5`; + document.getElementById('app-downloads').textContent = this.formatNumber(this.appData.downloads || 0); + + // Action buttons + const websiteBtn = document.getElementById('app-website'); + const githubBtn = document.getElementById('app-github'); + + if (this.appData.website_url) { + websiteBtn.href = this.appData.website_url; + } else { + websiteBtn.style.display = 'none'; + } + + if (this.appData.github_url) { + githubBtn.href = this.appData.github_url; + } else { + githubBtn.style.display = 'none'; + } + + // Contact + document.getElementById('app-contact').textContent = this.appData.contact_email || 'Not available'; + + // Integration guide + this.renderIntegrationGuide(); + } + + renderIntegrationGuide() { + // Installation code + const installCode = document.getElementById('install-code'); + if (this.appData.type === 'Open Source' && this.appData.github_url) { + installCode.textContent = `# Clone from GitHub +git clone ${this.appData.github_url} + +# Install dependencies +pip install -r requirements.txt`; + } else if (this.appData.name.toLowerCase().includes('api')) { + installCode.textContent = `# Install via pip +pip install ${this.appData.slug} + +# Or install from source +pip install git+${this.appData.github_url || 'https://github.com/example/repo'}`; + } + + // Usage code - customize based on category + const usageCode = document.getElementById('usage-code'); + if (this.appData.category === 'Browser Automation') { + usageCode.textContent = `from crawl4ai import AsyncWebCrawler +from ${this.appData.slug.replace(/-/g, '_')} import ${this.appData.name.replace(/\s+/g, '')} + +async def main(): + # Initialize ${this.appData.name} + automation = ${this.appData.name.replace(/\s+/g, '')}() + + async with AsyncWebCrawler() as crawler: + result = await crawler.arun( + url="https://example.com", + browser_config=automation.config, + wait_for="css:body" + ) + print(result.markdown)`; + } else if (this.appData.category === 'Proxy Services') { + usageCode.textContent = `from crawl4ai import AsyncWebCrawler +import ${this.appData.slug.replace(/-/g, '_')} + +# Configure proxy +proxy_config = { + "server": "${this.appData.website_url || 'https://proxy.example.com'}", + "username": "your_username", + "password": "your_password" +} + +async with AsyncWebCrawler(proxy=proxy_config) as crawler: + result = await crawler.arun( + url="https://example.com", + bypass_cache=True + ) + print(result.status_code)`; + } else if (this.appData.category === 'LLM Integration') { + usageCode.textContent = `from crawl4ai import AsyncWebCrawler +from crawl4ai.extraction_strategy import LLMExtractionStrategy + +# Configure LLM extraction +strategy = LLMExtractionStrategy( + provider="${this.appData.name.toLowerCase().includes('gpt') ? 'openai' : 'anthropic'}", + api_key="your-api-key", + model="${this.appData.name.toLowerCase().includes('gpt') ? 'gpt-4' : 'claude-3'}", + instruction="Extract structured data" +) + +async with AsyncWebCrawler() as crawler: + result = await crawler.arun( + url="https://example.com", + extraction_strategy=strategy + ) + print(result.extracted_content)`; + } + + // Integration example + const integrationCode = document.getElementById('integration-code'); + integrationCode.textContent = this.appData.integration_guide || +`# Complete ${this.appData.name} Integration Example + +from crawl4ai import AsyncWebCrawler +from crawl4ai.extraction_strategy import JsonCssExtractionStrategy +import json + +async def crawl_with_${this.appData.slug.replace(/-/g, '_')}(): + """ + Complete example showing how to use ${this.appData.name} + with Crawl4AI for production web scraping + """ + + # Define extraction schema + schema = { + "name": "ProductList", + "baseSelector": "div.product", + "fields": [ + {"name": "title", "selector": "h2", "type": "text"}, + {"name": "price", "selector": ".price", "type": "text"}, + {"name": "image", "selector": "img", "type": "attribute", "attribute": "src"}, + {"name": "link", "selector": "a", "type": "attribute", "attribute": "href"} + ] + } + + # Initialize crawler with ${this.appData.name} + async with AsyncWebCrawler( + browser_type="chromium", + headless=True, + verbose=True + ) as crawler: + + # Crawl with extraction + result = await crawler.arun( + url="https://example.com/products", + extraction_strategy=JsonCssExtractionStrategy(schema), + cache_mode="bypass", + wait_for="css:.product", + screenshot=True + ) + + # Process results + if result.success: + products = json.loads(result.extracted_content) + print(f"Found {len(products)} products") + + for product in products[:5]: + print(f"- {product['title']}: {product['price']}") + + return products + +# Run the crawler +if __name__ == "__main__": + import asyncio + asyncio.run(crawl_with_${this.appData.slug.replace(/-/g, '_')}())`; + } + + formatNumber(num) { + if (num >= 1000000) { + return (num / 1000000).toFixed(1) + 'M'; + } else if (num >= 1000) { + return (num / 1000).toFixed(1) + 'K'; + } + return num.toString(); + } + + setupEventListeners() { + // Tab switching + const tabs = document.querySelectorAll('.nav-tab'); + tabs.forEach(tab => { + tab.addEventListener('click', () => { + // Update active tab + tabs.forEach(t => t.classList.remove('active')); + tab.classList.add('active'); + + // Show corresponding content + const tabName = tab.dataset.tab; + document.querySelectorAll('.tab-content').forEach(content => { + content.classList.remove('active'); + }); + document.getElementById(`${tabName}-tab`).classList.add('active'); + }); + }); + + // Copy integration code + document.getElementById('copy-integration').addEventListener('click', () => { + const code = document.getElementById('integration-code').textContent; + navigator.clipboard.writeText(code).then(() => { + const btn = document.getElementById('copy-integration'); + const originalText = btn.innerHTML; + btn.innerHTML = '✓ Copied!'; + setTimeout(() => { + btn.innerHTML = originalText; + }, 2000); + }); + }); + + // Copy code buttons + document.querySelectorAll('.copy-btn').forEach(btn => { + btn.addEventListener('click', (e) => { + const codeBlock = e.target.closest('.code-block'); + const code = codeBlock.querySelector('code').textContent; + navigator.clipboard.writeText(code).then(() => { + btn.textContent = 'Copied!'; + setTimeout(() => { + btn.textContent = 'Copy'; + }, 2000); + }); + }); + }); + } + + async loadRelatedApps() { + try { + const response = await fetch(`${API_BASE}/apps?category=${encodeURIComponent(this.appData.category)}&limit=4`); + const apps = await response.json(); + + const relatedApps = apps.filter(app => app.slug !== this.appSlug).slice(0, 3); + const grid = document.getElementById('related-apps-grid'); + + grid.innerHTML = relatedApps.map(app => ` +
+

${app.name}

+

${app.description.substring(0, 100)}...

+
+ ${app.type} + ★ ${app.rating}/5 +
+
+ `).join(''); + } catch (error) { + console.error('Error loading related apps:', error); + } + } +} + +// Initialize when DOM is loaded +document.addEventListener('DOMContentLoaded', () => { + new AppDetailPage(); +}); \ No newline at end of file diff --git a/docs/md_v2/marketplace/frontend/app-detail.js b/docs/md_v2/marketplace/frontend/app-detail.js index 82422f14..d1b3b559 100644 --- a/docs/md_v2/marketplace/frontend/app-detail.js +++ b/docs/md_v2/marketplace/frontend/app-detail.js @@ -1,5 +1,5 @@ // App Detail Page JavaScript -const API_BASE = 'http://localhost:8100/api'; +const API_BASE = '/api'; class AppDetailPage { constructor() { diff --git a/docs/md_v2/marketplace/frontend/marketplace.js b/docs/md_v2/marketplace/frontend/marketplace.js index cdc22114..94a401bf 100644 --- a/docs/md_v2/marketplace/frontend/marketplace.js +++ b/docs/md_v2/marketplace/frontend/marketplace.js @@ -1,5 +1,5 @@ // Marketplace JS - Magazine Layout -const API_BASE = 'http://localhost:8100/api'; +const API_BASE = '/api'; const CACHE_TTL = 3600000; // 1 hour in ms class MarketplaceCache { diff --git a/docs/md_v2/marketplace/index.html b/docs/md_v2/marketplace/index.html new file mode 100644 index 00000000..c425420a --- /dev/null +++ b/docs/md_v2/marketplace/index.html @@ -0,0 +1,147 @@ + + + + + + Marketplace - Crawl4AI + + + +
+ +
+
+
+
+ +

+ [ + Marketplace + ] +

+
+

Tools, Integrations & Resources for Web Crawling

+
+
+ Apps: -- + Articles: -- + Downloads: -- +
+
+
+ + +
+
+ > + + / +
+
+ + +
+
+ + +
+ +
+ +
+ + +
+ +
+ + + + + +
+ +
+
+

> Latest Apps

+ +
+
+ +
+
+ + +
+
+

> Latest Articles

+
+
+ +
+
+ + +
+
+

# Trending

+
+ + +
+

+ Submit Your Tool

+

Share your integration

+ Submit → +
+
+
+ + +
+
+

> More Apps

+ +
+
+ +
+
+
+ + +
+
+
+

About Marketplace

+

Discover tools and integrations built by the Crawl4AI community.

+
+
+

Become a Sponsor

+

Reach developers building with Crawl4AI

+ +
+
+
+

[ Crawl4AI Marketplace · Updated -- ]

+
+
+
+ + + + \ No newline at end of file diff --git a/docs/md_v2/marketplace/marketplace.css b/docs/md_v2/marketplace/marketplace.css new file mode 100644 index 00000000..ad26c344 --- /dev/null +++ b/docs/md_v2/marketplace/marketplace.css @@ -0,0 +1,957 @@ +/* Marketplace CSS - Magazine Style Terminal Theme */ +@import url('../../assets/styles.css'); + +:root { + --primary-cyan: #50ffff; + --primary-teal: #09b5a5; + --accent-pink: #f380f5; + --bg-dark: #070708; + --bg-secondary: #1a1a1a; + --bg-tertiary: #3f3f44; + --text-primary: #e8e9ed; + --text-secondary: #d5cec0; + --text-tertiary: #a3abba; + --border-color: #3f3f44; + --success: #50ff50; + --error: #ff3c74; + --warning: #f59e0b; +} + +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +body { + font-family: 'Dank Mono', Monaco, monospace; + background: var(--bg-dark); + color: var(--text-primary); + line-height: 1.6; +} + +/* Global link styles */ +a { + color: var(--primary-cyan); + text-decoration: none; + transition: color 0.2s; +} + +a:hover { + color: var(--accent-pink); +} + +.marketplace-container { + min-height: 100vh; +} + +/* Header */ +.marketplace-header { + background: var(--bg-secondary); + border-bottom: 1px solid var(--border-color); + padding: 1.5rem 0; +} + +.header-content { + max-width: 1800px; + margin: 0 auto; + padding: 0 2rem; + display: flex; + justify-content: space-between; + align-items: center; +} + +.logo-title { + display: flex; + align-items: center; + gap: 1rem; +} + +.header-logo { + height: 40px; + width: auto; + filter: brightness(1.2); +} + +.marketplace-header h1 { + font-size: 1.5rem; + color: var(--primary-cyan); + margin: 0; +} + +.ascii-border { + color: var(--border-color); +} + +.tagline { + font-size: 0.875rem; + color: var(--text-tertiary); + margin-top: 0.25rem; +} + +.header-stats { + display: flex; + gap: 2rem; +} + +.stat-item { + font-size: 0.875rem; + color: var(--text-secondary); +} + +.stat-item span { + color: var(--primary-cyan); + font-weight: 600; +} + +/* Search and Filter Bar */ +.search-filter-bar { + max-width: 1800px; + margin: 1.5rem auto; + padding: 0 2rem; + display: flex; + gap: 1rem; + align-items: center; +} + +.search-box { + flex: 1; + max-width: 500px; + display: flex; + align-items: center; + background: var(--bg-secondary); + border: 1px solid var(--border-color); + padding: 0.75rem 1rem; + transition: border-color 0.2s; +} + +.search-box:focus-within { + border-color: var(--primary-cyan); +} + +.search-icon { + color: var(--text-tertiary); + margin-right: 1rem; +} + +#search-input { + flex: 1; + background: transparent; + border: none; + color: var(--text-primary); + font-family: inherit; + font-size: 0.9rem; + outline: none; +} + +.search-box kbd { + font-size: 0.75rem; + padding: 0.2rem 0.5rem; + background: var(--bg-tertiary); + border: 1px solid var(--border-color); + color: var(--text-tertiary); +} + +.category-filter { + display: flex; + gap: 0.5rem; + flex-wrap: wrap; +} + +.filter-btn { + background: transparent; + border: 1px solid var(--border-color); + color: var(--text-secondary); + padding: 0.5rem 1rem; + font-family: inherit; + font-size: 0.875rem; + cursor: pointer; + transition: all 0.2s; +} + +.filter-btn:hover { + border-color: var(--primary-cyan); + color: var(--primary-cyan); +} + +.filter-btn.active { + background: var(--primary-cyan); + color: var(--bg-dark); + border-color: var(--primary-cyan); +} + +/* Magazine Layout */ +.magazine-layout { + max-width: 1800px; + margin: 0 auto; + padding: 0 2rem 4rem; + display: grid; + grid-template-columns: 1fr; + gap: 2rem; +} + +/* Hero Featured Section */ +.hero-featured { + grid-column: 1 / -1; + position: relative; +} + +.hero-featured::before { + content: ''; + position: absolute; + top: -20px; + left: -20px; + right: -20px; + bottom: -20px; + background: radial-gradient(ellipse at center, rgba(80, 255, 255, 0.05), transparent 70%); + pointer-events: none; + z-index: -1; +} + +.featured-hero-card { + background: linear-gradient(135deg, #1a1a2e, #0f0f1e); + border: 2px solid var(--primary-cyan); + box-shadow: 0 0 30px rgba(80, 255, 255, 0.15), + inset 0 0 20px rgba(80, 255, 255, 0.05); + height: 380px; + position: relative; + overflow: hidden; + cursor: pointer; + transition: all 0.3s ease; + display: flex; + flex-direction: column; +} + +.featured-hero-card:hover { + border-color: var(--accent-pink); + box-shadow: 0 0 40px rgba(243, 128, 245, 0.2), + inset 0 0 30px rgba(243, 128, 245, 0.05); + transform: translateY(-2px); +} + +.hero-image { + width: 100%; + height: 240px; + background: linear-gradient(135deg, rgba(80, 255, 255, 0.1), rgba(243, 128, 245, 0.05)); + background-size: cover; + background-position: center; + display: flex; + align-items: center; + justify-content: center; + font-size: 3rem; + color: var(--primary-cyan); + flex-shrink: 0; + position: relative; + filter: brightness(1.1) contrast(1.1); +} + +.hero-image::after { + content: ''; + position: absolute; + bottom: 0; + left: 0; + right: 0; + height: 60%; + background: linear-gradient(to top, rgba(10, 10, 20, 0.95), transparent); +} + +.hero-content { + padding: 1.5rem; +} + +.hero-badge { + display: inline-block; + padding: 0.3rem 0.6rem; + background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal)); + color: var(--bg-dark); + font-size: 0.7rem; + text-transform: uppercase; + margin-bottom: 0.5rem; + font-weight: 600; + box-shadow: 0 2px 10px rgba(80, 255, 255, 0.3); +} + +.hero-title { + font-size: 1.6rem; + color: var(--primary-cyan); + margin: 0.5rem 0; + text-shadow: 0 0 20px rgba(80, 255, 255, 0.5); +} + +.hero-description { + color: var(--text-secondary); + line-height: 1.5; +} + +.hero-meta { + display: flex; + gap: 1.5rem; + margin-top: 1rem; + font-size: 0.875rem; +} + +.hero-meta span { + color: var(--text-tertiary); +} + +.hero-meta span:first-child { + color: var(--warning); +} + +/* Secondary Featured */ +.secondary-featured { + grid-column: 1 / -1; + height: 380px; + display: flex; + align-items: stretch; +} + +.featured-secondary-cards { + width: 100%; + display: flex; + flex-direction: column; + gap: 0.75rem; + justify-content: space-between; +} + +.secondary-card { + background: linear-gradient(135deg, rgba(80, 255, 255, 0.03), rgba(243, 128, 245, 0.02)); + border: 1px solid rgba(80, 255, 255, 0.3); + cursor: pointer; + transition: all 0.3s ease; + display: flex; + overflow: hidden; + height: calc((380px - 1.5rem) / 3); + flex: 1; + box-shadow: 0 2px 10px rgba(0, 0, 0, 0.3); +} + +.secondary-card:hover { + border-color: var(--accent-pink); + background: linear-gradient(135deg, rgba(243, 128, 245, 0.05), rgba(80, 255, 255, 0.03)); + box-shadow: 0 4px 15px rgba(243, 128, 245, 0.2); + transform: translateX(-3px); +} + +.secondary-image { + width: 120px; + background: linear-gradient(135deg, var(--bg-tertiary), var(--bg-secondary)); + background-size: cover; + background-position: center; + display: flex; + align-items: center; + justify-content: center; + font-size: 1.5rem; + color: var(--primary-cyan); + flex-shrink: 0; +} + +.secondary-content { + flex: 1; + padding: 1rem; + display: flex; + flex-direction: column; + justify-content: space-between; +} + +.secondary-title { + font-size: 1rem; + color: var(--text-primary); + margin-bottom: 0.25rem; +} + +.secondary-desc { + font-size: 0.75rem; + color: var(--text-secondary); + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; +} + +.secondary-meta { + font-size: 0.75rem; + color: var(--text-tertiary); +} + +.secondary-meta span:last-child { + color: var(--warning); +} + +/* Sponsored Section */ +.sponsored-section { + grid-column: 1 / -1; + background: var(--bg-secondary); + border: 1px solid var(--warning); + padding: 1rem; + position: relative; +} + +.section-label { + position: absolute; + top: -0.5rem; + left: 1rem; + background: var(--bg-secondary); + padding: 0 0.5rem; + color: var(--warning); + font-size: 0.65rem; + letter-spacing: 0.1em; +} + +.sponsored-cards { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + gap: 1rem; +} + +.sponsor-card { + padding: 1rem; + background: var(--bg-tertiary); + border: 1px solid var(--border-color); +} + +.sponsor-card h4 { + color: var(--accent-pink); + margin-bottom: 0.5rem; +} + +.sponsor-card p { + color: var(--text-secondary); + font-size: 0.85rem; + margin-bottom: 0.75rem; +} + +.sponsor-card a { + color: var(--primary-cyan); + text-decoration: none; + font-size: 0.85rem; +} + +.sponsor-card a:hover { + color: var(--accent-pink); +} + +/* Main Content Grid */ +.main-content { + grid-column: 1 / -1; + display: grid; + grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); + gap: 2rem; +} + +/* Column Headers */ +.column-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 1rem; + border-bottom: 1px solid var(--border-color); + padding-bottom: 0.5rem; +} + +.column-header h2 { + font-size: 1.1rem; + color: var(--text-primary); +} + +.mini-filter { + background: var(--bg-tertiary); + border: 1px solid var(--border-color); + color: var(--text-primary); + padding: 0.25rem 0.5rem; + font-family: inherit; + font-size: 0.75rem; +} + +.ascii-icon { + color: var(--primary-cyan); +} + +/* Apps Column */ +.apps-compact-grid { + display: flex; + flex-direction: column; + gap: 0.75rem; +} + +.app-compact { + background: var(--bg-secondary); + border: 1px solid var(--border-color); + border-left: 3px solid var(--border-color); + padding: 0.75rem; + cursor: pointer; + transition: all 0.2s; +} + +.app-compact:hover { + border-color: var(--primary-cyan); + border-left-color: var(--accent-pink); + transform: translateX(2px); +} + +.app-compact-header { + display: flex; + justify-content: space-between; + font-size: 0.75rem; + color: var(--text-tertiary); + margin-bottom: 0.25rem; +} + +.app-compact-header span:first-child { + color: var(--primary-cyan); +} + +.app-compact-header span:last-child { + color: var(--warning); +} + +.app-compact-title { + font-size: 0.9rem; + color: var(--text-primary); + margin-bottom: 0.25rem; +} + +.app-compact-desc { + font-size: 0.75rem; + color: var(--text-secondary); + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; +} + +/* Articles Column */ +.articles-compact-list { + display: flex; + flex-direction: column; + gap: 1rem; +} + +.article-compact { + border-left: 2px solid var(--border-color); + padding-left: 1rem; + cursor: pointer; + transition: all 0.2s; +} + +.article-compact:hover { + border-left-color: var(--primary-cyan); +} + +.article-meta { + font-size: 0.7rem; + color: var(--text-tertiary); + margin-bottom: 0.25rem; +} + +.article-meta span:first-child { + color: var(--accent-pink); +} + +.article-title { + font-size: 0.9rem; + color: var(--text-primary); + margin-bottom: 0.25rem; +} + +.article-author { + font-size: 0.75rem; + color: var(--text-secondary); +} + +/* Trending Column */ +.trending-items { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.trending-item { + display: flex; + align-items: center; + gap: 0.75rem; + padding: 0.5rem; + background: var(--bg-secondary); + border: 1px solid var(--border-color); + cursor: pointer; + transition: all 0.2s; +} + +.trending-item:hover { + border-color: var(--primary-cyan); +} + +.trending-rank { + font-size: 1.2rem; + color: var(--primary-cyan); + width: 2rem; + text-align: center; +} + +.trending-info { + flex: 1; +} + +.trending-name { + font-size: 0.85rem; + color: var(--text-primary); +} + +.trending-stats { + font-size: 0.7rem; + color: var(--text-tertiary); +} + +/* Submit Box */ +.submit-box { + margin-top: 1.5rem; + background: var(--bg-secondary); + border: 1px solid var(--primary-cyan); + padding: 1rem; + text-align: center; +} + +.submit-box h3 { + font-size: 1rem; + color: var(--primary-cyan); + margin-bottom: 0.5rem; +} + +.submit-box p { + font-size: 0.8rem; + color: var(--text-secondary); + margin-bottom: 0.75rem; +} + +.submit-btn { + display: inline-block; + padding: 0.5rem 1rem; + background: transparent; + border: 1px solid var(--primary-cyan); + color: var(--primary-cyan); + text-decoration: none; + transition: all 0.2s; +} + +.submit-btn:hover { + background: var(--primary-cyan); + color: var(--bg-dark); +} + +/* More Apps Section */ +.more-apps { + grid-column: 1 / -1; + margin-top: 2rem; +} + +.section-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 1rem; +} + +.more-apps-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); + gap: 1rem; +} + +.load-more-btn { + background: transparent; + border: 1px solid var(--border-color); + color: var(--text-secondary); + padding: 0.5rem 1.5rem; + font-family: inherit; + cursor: pointer; + transition: all 0.2s; +} + +.load-more-btn:hover { + border-color: var(--primary-cyan); + color: var(--primary-cyan); +} + +/* Footer */ +.marketplace-footer { + background: var(--bg-secondary); + border-top: 1px solid var(--border-color); + margin-top: 4rem; + padding: 2rem 0; +} + +.footer-content { + max-width: 1800px; + margin: 0 auto; + padding: 0 2rem; + display: grid; + grid-template-columns: 1fr 1fr; + gap: 2rem; +} + +.footer-section h3 { + font-size: 1rem; + margin-bottom: 0.5rem; + color: var(--primary-cyan); +} + +.footer-section p { + font-size: 0.875rem; + color: var(--text-secondary); + margin-bottom: 1rem; +} + +.sponsor-btn { + display: inline-block; + padding: 0.5rem 1rem; + background: transparent; + border: 1px solid var(--primary-cyan); + color: var(--primary-cyan); + text-decoration: none; + transition: all 0.2s; +} + +.sponsor-btn:hover { + background: var(--primary-cyan); + color: var(--bg-dark); +} + +.footer-bottom { + max-width: 1800px; + margin: 2rem auto 0; + padding: 1rem 2rem 0; + border-top: 1px solid var(--border-color); + font-size: 0.75rem; + color: var(--text-tertiary); +} + +/* Modal */ +.modal { + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: rgba(0, 0, 0, 0.8); + display: flex; + align-items: center; + justify-content: center; + z-index: 1000; +} + +.modal.hidden { + display: none; +} + +.modal-content { + background: var(--bg-secondary); + border: 1px solid var(--primary-cyan); + max-width: 800px; + width: 90%; + max-height: 80vh; + overflow-y: auto; + position: relative; +} + +.modal-close { + position: absolute; + top: 1rem; + right: 1rem; + background: transparent; + border: 1px solid var(--border-color); + color: var(--text-primary); + padding: 0.25rem 0.5rem; + cursor: pointer; + font-size: 1.2rem; +} + +.modal-close:hover { + border-color: var(--error); + color: var(--error); +} + +.app-detail { + padding: 2rem; +} + +.app-detail h2 { + font-size: 1.5rem; + margin-bottom: 1rem; + color: var(--primary-cyan); +} + +/* Loading */ +.loading { + text-align: center; + padding: 2rem; + color: var(--text-tertiary); +} + +.no-results { + text-align: center; + padding: 2rem; + color: var(--text-tertiary); +} + +/* Responsive - Tablet */ +@media (min-width: 768px) { + .magazine-layout { + grid-template-columns: repeat(2, 1fr); + } + + .hero-featured { + grid-column: 1 / -1; + } + + .secondary-featured { + grid-column: 1 / -1; + } + + .sponsored-section { + grid-column: 1 / -1; + } + + .main-content { + grid-column: 1 / -1; + grid-template-columns: repeat(2, 1fr); + } +} + +/* Responsive - Desktop */ +@media (min-width: 1024px) { + .magazine-layout { + grid-template-columns: repeat(3, 1fr); + } + + .hero-featured { + grid-column: 1 / 3; + grid-row: 1; + } + + .secondary-featured { + grid-column: 3 / 4; + grid-row: 1; + } + + .featured-secondary-cards { + flex-direction: column; + } + + .sponsored-section { + grid-column: 1 / -1; + } + + .main-content { + grid-column: 1 / -1; + grid-template-columns: repeat(3, 1fr); + } +} + +/* Responsive - Wide Desktop */ +@media (min-width: 1400px) { + .magazine-layout { + grid-template-columns: repeat(4, 1fr); + } + + .hero-featured { + grid-column: 1 / 3; + } + + .secondary-featured { + grid-column: 3 / 5; + grid-row: 1; + } + + .featured-secondary-cards { + grid-template-columns: repeat(2, 1fr); + } + + .main-content { + grid-template-columns: repeat(4, 1fr); + } + + .apps-column { + grid-column: span 2; + } + + .more-apps-grid { + grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); + } +} + +/* Responsive - Ultra Wide Desktop (for coders with wide monitors) */ +@media (min-width: 1800px) { + .magazine-layout { + grid-template-columns: repeat(5, 1fr); + } + + .hero-featured { + grid-column: 1 / 3; + } + + .secondary-featured { + grid-column: 3 / 6; + } + + .featured-secondary-cards { + grid-template-columns: repeat(3, 1fr); + } + + .sponsored-section { + grid-column: 1 / -1; + } + + .sponsored-cards { + grid-template-columns: repeat(5, 1fr); + } + + .main-content { + grid-template-columns: repeat(5, 1fr); + } + + .apps-column { + grid-column: span 2; + } + + .articles-column { + grid-column: span 2; + } + + .more-apps-grid { + grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); + } +} + +/* Responsive - Mobile */ +@media (max-width: 767px) { + .header-content { + flex-direction: column; + gap: 1rem; + } + + .search-filter-bar { + flex-direction: column; + align-items: stretch; + } + + .search-box { + max-width: none; + } + + .magazine-layout { + padding: 0 1rem 2rem; + } + + .footer-content { + grid-template-columns: 1fr; + } + + .secondary-card { + flex-direction: column; + } + + .secondary-image { + width: 100%; + height: 150px; + } +} \ No newline at end of file diff --git a/docs/md_v2/marketplace/marketplace.js b/docs/md_v2/marketplace/marketplace.js new file mode 100644 index 00000000..94a401bf --- /dev/null +++ b/docs/md_v2/marketplace/marketplace.js @@ -0,0 +1,395 @@ +// Marketplace JS - Magazine Layout +const API_BASE = '/api'; +const CACHE_TTL = 3600000; // 1 hour in ms + +class MarketplaceCache { + constructor() { + this.prefix = 'c4ai_market_'; + } + + get(key) { + const item = localStorage.getItem(this.prefix + key); + if (!item) return null; + + const data = JSON.parse(item); + if (Date.now() > data.expires) { + localStorage.removeItem(this.prefix + key); + return null; + } + return data.value; + } + + set(key, value, ttl = CACHE_TTL) { + const data = { + value: value, + expires: Date.now() + ttl + }; + localStorage.setItem(this.prefix + key, JSON.stringify(data)); + } + + clear() { + Object.keys(localStorage) + .filter(k => k.startsWith(this.prefix)) + .forEach(k => localStorage.removeItem(k)); + } +} + +class MarketplaceAPI { + constructor() { + this.cache = new MarketplaceCache(); + this.searchTimeout = null; + } + + async fetch(endpoint, useCache = true) { + const cacheKey = endpoint.replace(/[^\w]/g, '_'); + + if (useCache) { + const cached = this.cache.get(cacheKey); + if (cached) return cached; + } + + try { + const response = await fetch(`${API_BASE}${endpoint}`); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + + const data = await response.json(); + this.cache.set(cacheKey, data); + return data; + } catch (error) { + console.error('API Error:', error); + return null; + } + } + + async getStats() { + return this.fetch('/stats'); + } + + async getCategories() { + return this.fetch('/categories'); + } + + async getApps(params = {}) { + const query = new URLSearchParams(params).toString(); + return this.fetch(`/apps${query ? '?' + query : ''}`); + } + + async getArticles(params = {}) { + const query = new URLSearchParams(params).toString(); + return this.fetch(`/articles${query ? '?' + query : ''}`); + } + + async getSponsors() { + return this.fetch('/sponsors'); + } + + async search(query) { + if (query.length < 2) return {}; + return this.fetch(`/search?q=${encodeURIComponent(query)}`, false); + } +} + +class MarketplaceUI { + constructor() { + this.api = new MarketplaceAPI(); + this.currentCategory = 'all'; + this.currentType = ''; + this.searchTimeout = null; + this.loadedApps = 10; + this.init(); + } + + async init() { + await this.loadStats(); + await this.loadCategories(); + await this.loadFeaturedContent(); + await this.loadSponsors(); + await this.loadMainContent(); + this.setupEventListeners(); + } + + async loadStats() { + const stats = await this.api.getStats(); + if (stats) { + document.getElementById('total-apps').textContent = stats.total_apps || '0'; + document.getElementById('total-articles').textContent = stats.total_articles || '0'; + document.getElementById('total-downloads').textContent = stats.total_downloads || '0'; + document.getElementById('last-update').textContent = new Date().toLocaleDateString(); + } + } + + async loadCategories() { + const categories = await this.api.getCategories(); + if (!categories) return; + + const filter = document.getElementById('category-filter'); + categories.forEach(cat => { + const btn = document.createElement('button'); + btn.className = 'filter-btn'; + btn.dataset.category = cat.slug; + btn.textContent = cat.name; + btn.onclick = () => this.filterByCategory(cat.slug); + filter.appendChild(btn); + }); + } + + async loadFeaturedContent() { + // Load hero featured + const featured = await this.api.getApps({ featured: true, limit: 4 }); + if (!featured || !featured.length) return; + + // Hero card (first featured) + const hero = featured[0]; + const heroCard = document.getElementById('featured-hero'); + if (hero) { + const imageUrl = hero.image || ''; + heroCard.innerHTML = ` +
+ ${!imageUrl ? `[${hero.category || 'APP'}]` : ''} +
+
+ ${hero.type || 'PAID'} +

${hero.name}

+

${hero.description}

+
+ ★ ${hero.rating || 0}/5 + ${hero.downloads || 0} downloads +
+
+ `; + heroCard.onclick = () => this.showAppDetail(hero); + } + + // Secondary featured cards + const secondary = document.getElementById('featured-secondary'); + secondary.innerHTML = ''; + if (featured.length > 1) { + featured.slice(1, 4).forEach(app => { + const card = document.createElement('div'); + card.className = 'secondary-card'; + const imageUrl = app.image || ''; + card.innerHTML = ` +
+ ${!imageUrl ? `[${app.category || 'APP'}]` : ''} +
+
+

${app.name}

+

${(app.description || '').substring(0, 100)}...

+
+ ${app.type || 'Open Source'} · ★ ${app.rating || 0}/5 +
+
+ `; + card.onclick = () => this.showAppDetail(app); + secondary.appendChild(card); + }); + } + } + + async loadSponsors() { + const sponsors = await this.api.getSponsors(); + if (!sponsors || !sponsors.length) { + // Show placeholder if no sponsors + const container = document.getElementById('sponsored-content'); + container.innerHTML = ` + + `; + return; + } + + const container = document.getElementById('sponsored-content'); + container.innerHTML = sponsors.slice(0, 5).map(sponsor => ` + + `).join(''); + } + + async loadMainContent() { + // Load apps column + const apps = await this.api.getApps({ limit: 8 }); + if (apps && apps.length) { + const appsGrid = document.getElementById('apps-grid'); + appsGrid.innerHTML = apps.map(app => ` +
+
+ ${app.category} + ★ ${app.rating}/5 +
+
${app.name}
+
${app.description}
+
+ `).join(''); + } + + // Load articles column + const articles = await this.api.getArticles({ limit: 6 }); + if (articles && articles.length) { + const articlesList = document.getElementById('articles-list'); + articlesList.innerHTML = articles.map(article => ` +
+
+ ${article.category} · ${new Date(article.published_at).toLocaleDateString()} +
+
${article.title}
+
by ${article.author}
+
+ `).join(''); + } + + // Load trending + if (apps && apps.length) { + const trending = apps.slice(0, 5); + const trendingList = document.getElementById('trending-list'); + trendingList.innerHTML = trending.map((app, i) => ` +
+
${i + 1}
+
+
${app.name}
+
${app.downloads} downloads
+
+
+ `).join(''); + } + + // Load more apps grid + const moreApps = await this.api.getApps({ offset: 8, limit: 12 }); + if (moreApps && moreApps.length) { + const moreGrid = document.getElementById('more-apps-grid'); + moreGrid.innerHTML = moreApps.map(app => ` +
+
+ ${app.category} + ${app.type} +
+
${app.name}
+
+ `).join(''); + } + } + + setupEventListeners() { + // Search + const searchInput = document.getElementById('search-input'); + searchInput.addEventListener('input', (e) => { + clearTimeout(this.searchTimeout); + this.searchTimeout = setTimeout(() => this.search(e.target.value), 300); + }); + + // Keyboard shortcut + document.addEventListener('keydown', (e) => { + if (e.key === '/' && !searchInput.contains(document.activeElement)) { + e.preventDefault(); + searchInput.focus(); + } + if (e.key === 'Escape' && searchInput.contains(document.activeElement)) { + searchInput.blur(); + searchInput.value = ''; + } + }); + + // Type filter + const typeFilter = document.getElementById('type-filter'); + typeFilter.addEventListener('change', (e) => { + this.currentType = e.target.value; + this.loadMainContent(); + }); + + // Load more + const loadMore = document.getElementById('load-more'); + loadMore.addEventListener('click', () => this.loadMoreApps()); + } + + async filterByCategory(category) { + // Update active state + document.querySelectorAll('.filter-btn').forEach(btn => { + btn.classList.toggle('active', btn.dataset.category === category); + }); + + this.currentCategory = category; + await this.loadMainContent(); + } + + async search(query) { + if (!query) { + await this.loadMainContent(); + return; + } + + const results = await this.api.search(query); + if (!results) return; + + // Update apps grid with search results + if (results.apps && results.apps.length) { + const appsGrid = document.getElementById('apps-grid'); + appsGrid.innerHTML = results.apps.map(app => ` +
+
+ ${app.category} + ★ ${app.rating}/5 +
+
${app.name}
+
${app.description}
+
+ `).join(''); + } + + // Update articles with search results + if (results.articles && results.articles.length) { + const articlesList = document.getElementById('articles-list'); + articlesList.innerHTML = results.articles.map(article => ` +
+
+ ${article.category} · ${new Date(article.published_at).toLocaleDateString()} +
+
${article.title}
+
by ${article.author}
+
+ `).join(''); + } + } + + async loadMoreApps() { + this.loadedApps += 12; + const moreApps = await this.api.getApps({ offset: this.loadedApps, limit: 12 }); + if (moreApps && moreApps.length) { + const moreGrid = document.getElementById('more-apps-grid'); + moreApps.forEach(app => { + const card = document.createElement('div'); + card.className = 'app-compact'; + card.innerHTML = ` +
+ ${app.category} + ${app.type} +
+
${app.name}
+ `; + card.onclick = () => this.showAppDetail(app); + moreGrid.appendChild(card); + }); + } + } + + showAppDetail(app) { + // Navigate to detail page instead of showing modal + const slug = app.slug || app.name.toLowerCase().replace(/\s+/g, '-'); + window.location.href = `app-detail.html?app=${slug}`; + } + + showArticle(articleId) { + // Could create article detail page similarly + console.log('Show article:', articleId); + } +} + +// Initialize marketplace +let marketplace; +document.addEventListener('DOMContentLoaded', () => { + marketplace = new MarketplaceUI(); +}); \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index d39172f6..6406b028 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,7 +14,7 @@ nav: - "Demo Apps": "apps/index.md" - "C4A-Script Editor": "apps/c4a-script/index.html" - "LLM Context Builder": "apps/llmtxt/index.html" - - "Marketplace": "marketplace/frontend/index.html" + - "Marketplace": "marketplace/index.html" - "Marketplace Admin": "marketplace/admin/index.html" - Setup & Installation: - "Installation": "core/installation.md" From 5145d42df7539c2c0743494f50cd0c11b8b1cef7 Mon Sep 17 00:00:00 2001 From: unclecode Date: Fri, 3 Oct 2025 20:09:48 +0800 Subject: [PATCH 044/119] fix(docs): hide copy menu on non-markdown pages --- docs/md_v2/assets/page_actions.js | 301 ++++++++++++++++++------------ 1 file changed, 186 insertions(+), 115 deletions(-) diff --git a/docs/md_v2/assets/page_actions.js b/docs/md_v2/assets/page_actions.js index 92893730..984a2c09 100644 --- a/docs/md_v2/assets/page_actions.js +++ b/docs/md_v2/assets/page_actions.js @@ -7,9 +7,12 @@ document.addEventListener('DOMContentLoaded', () => { githubRepo: 'unclecode/crawl4ai', githubBranch: 'main', docsPath: 'docs/md_v2', - excludePaths: ['/apps/c4a-script/', '/apps/llmtxt/', '/apps/crawl4ai-assistant/'], // Don't show on app pages + excludePaths: ['/apps/c4a-script/', '/apps/llmtxt/', '/apps/crawl4ai-assistant/', '/core/ask-ai/'], // Don't show on app pages }; + let cachedMarkdown = null; + let cachedMarkdownPath = null; + // Check if we should show the button on this page function shouldShowButton() { const currentPath = window.location.pathname; @@ -19,6 +22,17 @@ document.addEventListener('DOMContentLoaded', () => { return false; } + // Don't show on 404 pages + if (document.title && document.title.toLowerCase().includes('404')) { + return false; + } + + // Require mkdocs main content container + const mainContent = document.getElementById('terminal-mkdocs-main-content'); + if (!mainContent) { + return false; + } + // Don't show on excluded paths (apps) for (const excludePath of config.excludePaths) { if (currentPath.includes(excludePath)) { @@ -53,6 +67,56 @@ document.addEventListener('DOMContentLoaded', () => { return `${path}.md`; } + async function loadMarkdownContent() { + const mdPath = getCurrentMarkdownPath(); + + if (!mdPath) { + throw new Error('Invalid markdown path'); + } + + const rawUrl = getGithubRawUrl(); + const response = await fetch(rawUrl); + + if (!response.ok) { + throw new Error(`Failed to fetch markdown: ${response.status}`); + } + + const markdown = await response.text(); + cachedMarkdown = markdown; + cachedMarkdownPath = mdPath; + return markdown; + } + + async function ensureMarkdownCached() { + const mdPath = getCurrentMarkdownPath(); + + if (!mdPath) { + return false; + } + + if (cachedMarkdown && cachedMarkdownPath === mdPath) { + return true; + } + + try { + await loadMarkdownContent(); + return true; + } catch (error) { + console.warn('Page Actions: Markdown not available for this page.', error); + cachedMarkdown = null; + cachedMarkdownPath = null; + return false; + } + } + + async function getMarkdownContent() { + const available = await ensureMarkdownCached(); + if (!available) { + throw new Error('Markdown not available for this page.'); + } + return cachedMarkdown; + } + // Get GitHub raw URL for current page function getGithubRawUrl() { const mdPath = getCurrentMarkdownPath(); @@ -180,19 +244,11 @@ document.addEventListener('DOMContentLoaded', () => { // Copy markdown to clipboard async function copyMarkdownToClipboard(link) { - const rawUrl = getGithubRawUrl(); - // Add loading state link.classList.add('loading'); try { - const response = await fetch(rawUrl); - - if (!response.ok) { - throw new Error(`Failed to fetch markdown: ${response.status}`); - } - - const markdown = await response.text(); + const markdown = await getMarkdownContent(); // Copy to clipboard await navigator.clipboard.writeText(markdown); @@ -221,126 +277,141 @@ document.addEventListener('DOMContentLoaded', () => { window.open(githubUrl, '_blank', 'noopener,noreferrer'); } - // Initialize - const { button, dropdown, overlay } = createPageActionsUI(); - - // Event listeners - button.addEventListener('click', (e) => { - e.stopPropagation(); - toggleDropdown(button, dropdown, overlay); - }); - - overlay.addEventListener('click', () => { - closeDropdown(button, dropdown, overlay); - }); - - // Copy markdown action - document.getElementById('action-copy-markdown').addEventListener('click', async (e) => { - e.preventDefault(); - e.stopPropagation(); - await copyMarkdownToClipboard(e.currentTarget); - }); - - // View markdown action - document.getElementById('action-view-markdown').addEventListener('click', (e) => { - e.preventDefault(); - e.stopPropagation(); - viewMarkdown(); - closeDropdown(button, dropdown, overlay); - }); - - // Ask AI action (disabled for now) - document.getElementById('action-ask-ai').addEventListener('click', (e) => { - e.preventDefault(); - e.stopPropagation(); - // Future: Integrate with Ask AI feature - // For now, do nothing (disabled state) - }); - - // Close on ESC key - document.addEventListener('keydown', (e) => { - if (e.key === 'Escape' && dropdown.classList.contains('active')) { - closeDropdown(button, dropdown, overlay); + (async () => { + if (!shouldShowButton()) { + return; } - }); - // Close when clicking outside - document.addEventListener('click', (e) => { - if (!dropdown.contains(e.target) && !button.contains(e.target)) { - closeDropdown(button, dropdown, overlay); + const markdownAvailable = await ensureMarkdownCached(); + if (!markdownAvailable) { + return; } - }); - // Prevent dropdown from closing when clicking inside - dropdown.addEventListener('click', (e) => { - // Only stop propagation if not clicking on a link - if (!e.target.closest('.page-action-link')) { + const ui = createPageActionsUI(); + if (!ui) { + return; + } + + const { button, dropdown, overlay } = ui; + + // Event listeners + button.addEventListener('click', (e) => { e.stopPropagation(); - } - }); - - // Close dropdown on link click (except for copy which handles itself) - dropdown.querySelectorAll('.page-action-link:not(#action-copy-markdown)').forEach(link => { - link.addEventListener('click', () => { - if (!link.classList.contains('disabled')) { - setTimeout(() => { - closeDropdown(button, dropdown, overlay); - }, 100); - } + toggleDropdown(button, dropdown, overlay); }); - }); - // Handle window resize - let resizeTimer; - window.addEventListener('resize', () => { - clearTimeout(resizeTimer); - resizeTimer = setTimeout(() => { - // Close dropdown on resize to prevent positioning issues - if (dropdown.classList.contains('active')) { + overlay.addEventListener('click', () => { + closeDropdown(button, dropdown, overlay); + }); + + // Copy markdown action + document.getElementById('action-copy-markdown').addEventListener('click', async (e) => { + e.preventDefault(); + e.stopPropagation(); + await copyMarkdownToClipboard(e.currentTarget); + }); + + // View markdown action + document.getElementById('action-view-markdown').addEventListener('click', (e) => { + e.preventDefault(); + e.stopPropagation(); + viewMarkdown(); + closeDropdown(button, dropdown, overlay); + }); + + // Ask AI action (disabled for now) + document.getElementById('action-ask-ai').addEventListener('click', (e) => { + e.preventDefault(); + e.stopPropagation(); + // Future: Integrate with Ask AI feature + // For now, do nothing (disabled state) + }); + + // Close on ESC key + document.addEventListener('keydown', (e) => { + if (e.key === 'Escape' && dropdown.classList.contains('active')) { closeDropdown(button, dropdown, overlay); } - }, 250); - }); + }); - // Accessibility: Focus management - button.addEventListener('keydown', (e) => { - if (e.key === 'Enter' || e.key === ' ') { - e.preventDefault(); - toggleDropdown(button, dropdown, overlay); + // Close when clicking outside + document.addEventListener('click', (e) => { + if (!dropdown.contains(e.target) && !button.contains(e.target)) { + closeDropdown(button, dropdown, overlay); + } + }); - // Focus first menu item when opening - if (dropdown.classList.contains('active')) { - const firstLink = dropdown.querySelector('.page-action-link:not(.disabled)'); - if (firstLink) { - setTimeout(() => firstLink.focus(), 100); + // Prevent dropdown from closing when clicking inside + dropdown.addEventListener('click', (e) => { + // Only stop propagation if not clicking on a link + if (!e.target.closest('.page-action-link')) { + e.stopPropagation(); + } + }); + + // Close dropdown on link click (except for copy which handles itself) + dropdown.querySelectorAll('.page-action-link:not(#action-copy-markdown)').forEach(link => { + link.addEventListener('click', () => { + if (!link.classList.contains('disabled')) { + setTimeout(() => { + closeDropdown(button, dropdown, overlay); + }, 100); + } + }); + }); + + // Handle window resize + let resizeTimer; + window.addEventListener('resize', () => { + clearTimeout(resizeTimer); + resizeTimer = setTimeout(() => { + // Close dropdown on resize to prevent positioning issues + if (dropdown.classList.contains('active')) { + closeDropdown(button, dropdown, overlay); + } + }, 250); + }); + + // Accessibility: Focus management + button.addEventListener('keydown', (e) => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + toggleDropdown(button, dropdown, overlay); + + // Focus first menu item when opening + if (dropdown.classList.contains('active')) { + const firstLink = dropdown.querySelector('.page-action-link:not(.disabled)'); + if (firstLink) { + setTimeout(() => firstLink.focus(), 100); + } } } - } - }); + }); - // Arrow key navigation within menu - dropdown.addEventListener('keydown', (e) => { - if (!dropdown.classList.contains('active')) return; + // Arrow key navigation within menu + dropdown.addEventListener('keydown', (e) => { + if (!dropdown.classList.contains('active')) return; - const links = Array.from(dropdown.querySelectorAll('.page-action-link:not(.disabled)')); - const currentIndex = links.indexOf(document.activeElement); + const links = Array.from(dropdown.querySelectorAll('.page-action-link:not(.disabled)')); + const currentIndex = links.indexOf(document.activeElement); - if (e.key === 'ArrowDown') { - e.preventDefault(); - const nextIndex = (currentIndex + 1) % links.length; - links[nextIndex].focus(); - } else if (e.key === 'ArrowUp') { - e.preventDefault(); - const prevIndex = (currentIndex - 1 + links.length) % links.length; - links[prevIndex].focus(); - } else if (e.key === 'Home') { - e.preventDefault(); - links[0].focus(); - } else if (e.key === 'End') { - e.preventDefault(); - links[links.length - 1].focus(); - } - }); + if (e.key === 'ArrowDown') { + e.preventDefault(); + const nextIndex = (currentIndex + 1) % links.length; + links[nextIndex].focus(); + } else if (e.key === 'ArrowUp') { + e.preventDefault(); + const prevIndex = (currentIndex - 1 + links.length) % links.length; + links[prevIndex].focus(); + } else if (e.key === 'Home') { + e.preventDefault(); + links[0].focus(); + } else if (e.key === 'End') { + e.preventDefault(); + links[links.length - 1].focus(); + } + }); - console.log('Page Actions initialized for:', getCurrentMarkdownPath()); + console.log('Page Actions initialized for:', getCurrentMarkdownPath()); + })(); }); \ No newline at end of file From 7dfe528d43670163faa5cd89d47520753f44a12d Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Fri, 3 Oct 2025 22:00:46 +0530 Subject: [PATCH 045/119] fix(docs): standardize C4A-Script tutorial, add CLI identity-based crawling, and add sponsorship CTA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Switch installs to pip install -r requirements.txt (tutorial and app docs) - Update local run steps to python server.py and http://localhost:8000 - Set default PORT to 8000; update port-in-use commands and alt port 8001 - Replace unsupported :contains() example with accessible attribute selector - Update example URLs in tutorial servers to 127.0.0.1:8000 - Add “Identity-based crawling” section with crwl profiles CLI workflow and code usage - Replace legacy-docs note with sponsorship message in docs/md_v2/index.md - Minor copy and consistency fixes across pages --- docs/examples/c4a_script/tutorial/README.md | 10 +++--- docs/examples/c4a_script/tutorial/server.py | 2 +- .../md_v2/advanced/identity-based-crawling.md | 36 +++++++++++++++++++ docs/md_v2/apps/c4a-script/README.md | 10 +++--- docs/md_v2/apps/c4a-script/server.py | 4 +-- docs/md_v2/core/c4a-script.md | 10 +++--- docs/md_v2/index.md | 2 +- 7 files changed, 55 insertions(+), 19 deletions(-) diff --git a/docs/examples/c4a_script/tutorial/README.md b/docs/examples/c4a_script/tutorial/README.md index 81f855ee..2d6940bb 100644 --- a/docs/examples/c4a_script/tutorial/README.md +++ b/docs/examples/c4a_script/tutorial/README.md @@ -18,7 +18,7 @@ A comprehensive web-based tutorial for learning and experimenting with C4A-Scrip 2. **Install Dependencies** ```bash - pip install flask + pip install -r requirements.txt ``` 3. **Launch the Server** @@ -28,7 +28,7 @@ A comprehensive web-based tutorial for learning and experimenting with C4A-Scrip 4. **Open in Browser** ``` - http://localhost:8080 + http://localhost:8000 ``` **🌐 Try Online**: [Live Demo](https://docs.crawl4ai.com/c4a-script/demo) @@ -325,7 +325,7 @@ Powers the recording functionality: ### Configuration ```python # server.py configuration -PORT = 8080 +PORT = 8000 DEBUG = True THREADED = True ``` @@ -343,9 +343,9 @@ THREADED = True **Port Already in Use** ```bash # Kill existing process -lsof -ti:8080 | xargs kill -9 +lsof -ti:8000 | xargs kill -9 # Or use different port -python server.py --port 8081 +python server.py --port 8001 ``` **Blockly Not Loading** diff --git a/docs/examples/c4a_script/tutorial/server.py b/docs/examples/c4a_script/tutorial/server.py index f9cb81e9..2537e4c3 100644 --- a/docs/examples/c4a_script/tutorial/server.py +++ b/docs/examples/c4a_script/tutorial/server.py @@ -216,7 +216,7 @@ def get_examples(): 'name': 'Handle Cookie Banner', 'description': 'Accept cookies and close newsletter popup', 'script': '''# Handle cookie banner and newsletter -GO http://127.0.0.1:8080/playground/ +GO http://127.0.0.1:8000/playground/ WAIT `body` 2 IF (EXISTS `.cookie-banner`) THEN CLICK `.accept` IF (EXISTS `.newsletter-popup`) THEN CLICK `.close`''' diff --git a/docs/md_v2/advanced/identity-based-crawling.md b/docs/md_v2/advanced/identity-based-crawling.md index 3864f840..2b155857 100644 --- a/docs/md_v2/advanced/identity-based-crawling.md +++ b/docs/md_v2/advanced/identity-based-crawling.md @@ -82,6 +82,42 @@ If you installed Crawl4AI (which installs Playwright under the hood), you alread --- +### Creating a Profile Using the Crawl4AI CLI (Easiest) + +If you prefer a guided, interactive setup, use the built-in CLI to create and manage persistent browser profiles. + +1.⠀Launch the profile manager: + ```bash + crwl profiles + ``` + +2.⠀Choose "Create new profile" and enter a profile name. A Chromium window opens so you can log in to sites and configure settings. When finished, return to the terminal and press `q` to save the profile. + +3.⠀Profiles are saved under `~/.crawl4ai/profiles/` (for example: `/home//.crawl4ai/profiles/test_profile_1`) along with a `storage_state.json` for cookies and session data. + +4.⠀Optionally, choose "List profiles" in the CLI to view available profiles and their paths. + +5.⠀Use the saved path with `BrowserConfig.user_data_dir`: + ```python + from crawl4ai import AsyncWebCrawler, BrowserConfig + + profile_path = "/home//.crawl4ai/profiles/test_profile_1" + + browser_config = BrowserConfig( + headless=True, + use_managed_browser=True, + user_data_dir=profile_path, + browser_type="chromium", + ) + + async with AsyncWebCrawler(config=browser_config) as crawler: + result = await crawler.arun(url="https://example.com/private") + ``` + +The CLI also supports listing and deleting profiles, and even testing a crawl directly from the menu. + +--- + ## 3. Using Managed Browsers in Crawl4AI Once you have a data directory with your session data, pass it to **`BrowserConfig`**: diff --git a/docs/md_v2/apps/c4a-script/README.md b/docs/md_v2/apps/c4a-script/README.md index 81f855ee..2d6940bb 100644 --- a/docs/md_v2/apps/c4a-script/README.md +++ b/docs/md_v2/apps/c4a-script/README.md @@ -18,7 +18,7 @@ A comprehensive web-based tutorial for learning and experimenting with C4A-Scrip 2. **Install Dependencies** ```bash - pip install flask + pip install -r requirements.txt ``` 3. **Launch the Server** @@ -28,7 +28,7 @@ A comprehensive web-based tutorial for learning and experimenting with C4A-Scrip 4. **Open in Browser** ``` - http://localhost:8080 + http://localhost:8000 ``` **🌐 Try Online**: [Live Demo](https://docs.crawl4ai.com/c4a-script/demo) @@ -325,7 +325,7 @@ Powers the recording functionality: ### Configuration ```python # server.py configuration -PORT = 8080 +PORT = 8000 DEBUG = True THREADED = True ``` @@ -343,9 +343,9 @@ THREADED = True **Port Already in Use** ```bash # Kill existing process -lsof -ti:8080 | xargs kill -9 +lsof -ti:8000 | xargs kill -9 # Or use different port -python server.py --port 8081 +python server.py --port 8001 ``` **Blockly Not Loading** diff --git a/docs/md_v2/apps/c4a-script/server.py b/docs/md_v2/apps/c4a-script/server.py index 6242789d..2537e4c3 100644 --- a/docs/md_v2/apps/c4a-script/server.py +++ b/docs/md_v2/apps/c4a-script/server.py @@ -216,7 +216,7 @@ def get_examples(): 'name': 'Handle Cookie Banner', 'description': 'Accept cookies and close newsletter popup', 'script': '''# Handle cookie banner and newsletter -GO http://127.0.0.1:8080/playground/ +GO http://127.0.0.1:8000/playground/ WAIT `body` 2 IF (EXISTS `.cookie-banner`) THEN CLICK `.accept` IF (EXISTS `.newsletter-popup`) THEN CLICK `.close`''' @@ -283,7 +283,7 @@ WAIT `.success-message` 5''' return jsonify(examples) if __name__ == '__main__': - port = int(os.environ.get('PORT', 8080)) + port = int(os.environ.get('PORT', 8000)) print(f""" ╔══════════════════════════════════════════════════════════╗ ║ C4A-Script Interactive Tutorial Server ║ diff --git a/docs/md_v2/core/c4a-script.md b/docs/md_v2/core/c4a-script.md index d92e426e..1af3da4e 100644 --- a/docs/md_v2/core/c4a-script.md +++ b/docs/md_v2/core/c4a-script.md @@ -69,12 +69,12 @@ The tutorial includes a Flask-based web interface with: cd docs/examples/c4a_script/tutorial/ # Install dependencies -pip install flask +pip install -r requirements.txt # Launch the tutorial server -python app.py +python server.py -# Open http://localhost:5000 in your browser +# Open http://localhost:8000 in your browser ``` ## Core Concepts @@ -111,8 +111,8 @@ CLICK `.submit-btn` # By attribute CLICK `button[type="submit"]` -# By text content -CLICK `button:contains("Sign In")` +# By accessible attributes +CLICK `button[aria-label="Search"][title="Search"]` # Complex selectors CLICK `.form-container input[name="email"]` diff --git a/docs/md_v2/index.md b/docs/md_v2/index.md index d497ca89..e7566e7b 100644 --- a/docs/md_v2/index.md +++ b/docs/md_v2/index.md @@ -57,7 +57,7 @@ Crawl4AI is the #1 trending GitHub repository, actively maintained by a vibrant community. It delivers blazing-fast, AI-ready web crawling tailored for large language models, AI agents, and data pipelines. Fully open source, flexible, and built for real-time performance, **Crawl4AI** empowers developers with unmatched speed, precision, and deployment ease. -> **Note**: If you're looking for the old documentation, you can access it [here](https://old.docs.crawl4ai.com). +> Enjoy using Crawl4AI? Consider **[becoming a sponsor](https://github.com/sponsors/unclecode)** to support ongoing development and community growth! ## 🎯 New: Adaptive Web Crawling From 46e1a67f614590e874c53fe31ad9dc333e2750c6 Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Mon, 6 Oct 2025 14:38:38 +0530 Subject: [PATCH 046/119] fix(docker): Remove environment variable overrides in docker-compose.yml (#1411) The docker-compose.yml had an `environment:` section with variable substitutions (${VAR:-}) that was overriding values from .llm.env with empty strings. - Commented out the `environment:` section to prevent overwrites - Added clear warning comment explaining the override behavior - .llm.env values now load directly into container without interference --- docker-compose.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 100d6973..cb99c18f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,15 +6,16 @@ x-base-config: &base-config - "11235:11235" # Gunicorn port env_file: - .llm.env # API keys (create from .llm.env.example) - environment: - - OPENAI_API_KEY=${OPENAI_API_KEY:-} - - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-} - - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} - - GROQ_API_KEY=${GROQ_API_KEY:-} - - TOGETHER_API_KEY=${TOGETHER_API_KEY:-} - - MISTRAL_API_KEY=${MISTRAL_API_KEY:-} - - GEMINI_API_TOKEN=${GEMINI_API_TOKEN:-} - - LLM_PROVIDER=${LLM_PROVIDER:-} # Optional: Override default provider (e.g., "anthropic/claude-3-opus") + # Uncomment to set default environment variables (will overwrite .llm.env) + # environment: + # - OPENAI_API_KEY=${OPENAI_API_KEY:-} + # - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-} + # - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} + # - GROQ_API_KEY=${GROQ_API_KEY:-} + # - TOGETHER_API_KEY=${TOGETHER_API_KEY:-} + # - MISTRAL_API_KEY=${MISTRAL_API_KEY:-} + # - GEMINI_API_KEY=${GEMINI_API_KEY:-} + # - LLM_PROVIDER=${LLM_PROVIDER:-} # Optional: Override default provider (e.g., "anthropic/claude-3-opus") volumes: - /dev/shm:/dev/shm # Chromium performance deploy: From 8c622777181bdc612c13904f9d35ef08814fd4c0 Mon Sep 17 00:00:00 2001 From: unclecode Date: Mon, 6 Oct 2025 20:58:35 +0800 Subject: [PATCH 047/119] feat(marketplace): add sponsor logo uploads Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- docs/md_v2/marketplace/admin/admin.css | 109 +++++++++++++ docs/md_v2/marketplace/admin/admin.js | 143 ++++++++++++++++-- docs/md_v2/marketplace/backend/server.py | 46 +++++- .../marketplace/backend/uploads/.gitignore | 2 + docs/md_v2/marketplace/marketplace.css | 15 ++ docs/md_v2/marketplace/marketplace.js | 19 ++- 6 files changed, 321 insertions(+), 13 deletions(-) create mode 100644 docs/md_v2/marketplace/backend/uploads/.gitignore diff --git a/docs/md_v2/marketplace/admin/admin.css b/docs/md_v2/marketplace/admin/admin.css index 7296a801..66b975a9 100644 --- a/docs/md_v2/marketplace/admin/admin.css +++ b/docs/md_v2/marketplace/admin/admin.css @@ -431,6 +431,16 @@ gap: 0.5rem; } +.table-logo { + width: 48px; + height: 48px; + object-fit: contain; + border-radius: 6px; + background: var(--bg-tertiary); + border: 1px solid var(--border-color); + padding: 4px; +} + .btn-edit, .btn-delete, .btn-duplicate { padding: 0.25rem 0.5rem; background: transparent; @@ -585,6 +595,105 @@ cursor: pointer; } +.sponsor-form { + grid-template-columns: 200px repeat(2, minmax(220px, 1fr)); + align-items: flex-start; + grid-auto-flow: dense; +} + +.sponsor-logo-group { + grid-row: span 3; + display: flex; + flex-direction: column; + gap: 0.75rem; +} + +.span-two { + grid-column: span 2; +} + +.logo-upload { + position: relative; + width: 180px; +} + +.image-preview { + width: 180px; + height: 180px; + border: 1px dashed var(--border-color); + border-radius: 12px; + display: flex; + align-items: center; + justify-content: center; + background: var(--bg-tertiary); + overflow: hidden; +} + +.image-preview.empty { + color: var(--text-secondary); + font-size: 0.75rem; + text-align: center; + padding: 0.75rem; +} + +.image-preview img { + max-width: 100%; + max-height: 100%; + object-fit: contain; +} + +.upload-btn { + position: absolute; + left: 50%; + bottom: 12px; + transform: translateX(-50%); + padding: 0.35rem 1rem; + background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal)); + border: none; + border-radius: 999px; + color: var(--bg-dark); + font-size: 0.75rem; + font-weight: 600; + cursor: pointer; + box-shadow: 0 6px 18px rgba(80, 255, 255, 0.25); +} + +.upload-btn:hover { + box-shadow: 0 8px 22px rgba(80, 255, 255, 0.35); +} + +.logo-upload input[type="file"] { + display: none; +} + +.upload-hint { + font-size: 0.75rem; + color: var(--text-secondary); + margin: 0; +} + +@media (max-width: 960px) { + .sponsor-form { + grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); + } + + .sponsor-logo-group { + grid-column: 1 / -1; + grid-row: auto; + flex-direction: row; + align-items: center; + gap: 1.5rem; + } + + .logo-upload { + width: 160px; + } + + .span-two { + grid-column: 1 / -1; + } +} + /* Rich Text Editor */ .editor-toolbar { display: flex; diff --git a/docs/md_v2/marketplace/admin/admin.js b/docs/md_v2/marketplace/admin/admin.js index 258858da..adb31d0b 100644 --- a/docs/md_v2/marketplace/admin/admin.js +++ b/docs/md_v2/marketplace/admin/admin.js @@ -1,5 +1,21 @@ // Admin Dashboard - Smart & Powerful -const API_BASE = '/api'; +const { API_BASE, API_ORIGIN } = (() => { + const { hostname, port } = window.location; + if ((hostname === 'localhost' || hostname === '127.0.0.1') && port === '8000') { + const origin = 'http://127.0.0.1:8100'; + return { API_BASE: `${origin}/api`, API_ORIGIN: origin }; + } + return { API_BASE: '/api', API_ORIGIN: '' }; +})(); + +const resolveAssetUrl = (path) => { + if (!path) return ''; + if (/^https?:\/\//i.test(path)) return path; + if (path.startsWith('/') && API_ORIGIN) { + return `${API_ORIGIN}${path}`; + } + return path; +}; class AdminDashboard { constructor() { @@ -144,13 +160,19 @@ class AdminDashboard { } async apiCall(endpoint, options = {}) { + const isFormData = options.body instanceof FormData; + const headers = { + 'Authorization': `Bearer ${this.token}`, + ...options.headers + }; + + if (!isFormData && !headers['Content-Type']) { + headers['Content-Type'] = 'application/json'; + } + const response = await fetch(`${API_BASE}${endpoint}`, { ...options, - headers: { - 'Authorization': `Bearer ${this.token}`, - 'Content-Type': 'application/json', - ...options.headers - } + headers }); if (response.status === 401) { @@ -189,7 +211,10 @@ class AdminDashboard { } async loadSponsors() { - this.data.sponsors = await this.apiCall('/sponsors'); + const cacheBuster = Date.now(); + this.data.sponsors = await this.apiCall(`/sponsors?limit=100&_=${cacheBuster}`, { + cache: 'no-store' + }); this.renderSponsorsTable(this.data.sponsors); } @@ -314,6 +339,7 @@ class AdminDashboard { ID + Logo Company Tier Start @@ -326,6 +352,7 @@ class AdminDashboard { ${sponsors.map(sponsor => ` ${sponsor.id} + ${sponsor.logo_url ? `` : '-'} ${sponsor.company_name} ${sponsor.tier} ${new Date(sponsor.start_date).toLocaleDateString()} @@ -389,6 +416,10 @@ class AdminDashboard { modal.classList.remove('hidden'); modal.dataset.type = type; + + if (type === 'sponsors') { + this.setupLogoUploadHandlers(); + } } getAppForm(app) { @@ -524,9 +555,22 @@ class AdminDashboard { } getSponsorForm(sponsor) { + const existingFile = sponsor?.logo_url ? sponsor.logo_url.split('/').pop().split('?')[0] : ''; return ` -
-
+ - + \ No newline at end of file diff --git a/docs/md_v2/marketplace/backend/server.py b/docs/md_v2/marketplace/backend/server.py index 77540020..0f177386 100644 --- a/docs/md_v2/marketplace/backend/server.py +++ b/docs/md_v2/marketplace/backend/server.py @@ -7,6 +7,7 @@ from typing import Optional, Dict, Any import json import hashlib import secrets +import re from pathlib import Path from database import DatabaseManager from datetime import datetime, timedelta @@ -58,6 +59,29 @@ def json_response(data, cache_time=3600): } ) + +def to_int(value, default=0): + """Coerce incoming values to integers, falling back to default.""" + if value is None: + return default + if isinstance(value, bool): + return int(value) + if isinstance(value, (int, float)): + return int(value) + + if isinstance(value, str): + stripped = value.strip() + if not stripped: + return default + + match = re.match(r"^-?\d+", stripped) + if match: + try: + return int(match.group()) + except ValueError: + return default + return default + # ============= PUBLIC ENDPOINTS ============= @app.get("/api/apps") @@ -141,6 +165,8 @@ async def get_article(slug: str): async def get_categories(): """Get all categories ordered by index""" categories = db.get_all('categories', limit=50) + for category in categories: + category['order_index'] = to_int(category.get('order_index'), 0) categories.sort(key=lambda x: x.get('order_index', 0)) return json_response(categories, cache_time=7200) @@ -360,6 +386,9 @@ async def delete_article(article_id: int): async def create_category(category_data: Dict[str, Any]): """Create new category""" try: + category_data = dict(category_data) + category_data['order_index'] = to_int(category_data.get('order_index'), 0) + cursor = db.conn.cursor() columns = ', '.join(category_data.keys()) placeholders = ', '.join(['?' for _ in category_data]) @@ -374,6 +403,10 @@ async def create_category(category_data: Dict[str, Any]): async def update_category(cat_id: int, category_data: Dict[str, Any]): """Update category""" try: + category_data = dict(category_data) + if 'order_index' in category_data: + category_data['order_index'] = to_int(category_data.get('order_index'), 0) + set_clause = ', '.join([f"{k} = ?" for k in category_data.keys()]) cursor = db.conn.cursor() cursor.execute(f"UPDATE categories SET {set_clause} WHERE id = ?", @@ -383,6 +416,18 @@ async def update_category(cat_id: int, category_data: Dict[str, Any]): except Exception as e: raise HTTPException(status_code=400, detail=str(e)) + +@app.delete("/api/admin/categories/{cat_id}", dependencies=[Depends(verify_token)]) +async def delete_category(cat_id: int): + """Delete category""" + try: + cursor = db.conn.cursor() + cursor.execute("DELETE FROM categories WHERE id = ?", (cat_id,)) + db.conn.commit() + return {"message": "Category deleted"} + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + # Sponsors CRUD @app.post("/api/admin/sponsors", dependencies=[Depends(verify_token)]) async def create_sponsor(sponsor_data: Dict[str, Any]): @@ -411,6 +456,18 @@ async def update_sponsor(sponsor_id: int, sponsor_data: Dict[str, Any]): except Exception as e: raise HTTPException(status_code=400, detail=str(e)) + +@app.delete("/api/admin/sponsors/{sponsor_id}", dependencies=[Depends(verify_token)]) +async def delete_sponsor(sponsor_id: int): + """Delete sponsor""" + try: + cursor = db.conn.cursor() + cursor.execute("DELETE FROM sponsors WHERE id = ?", (sponsor_id,)) + db.conn.commit() + return {"message": "Sponsor deleted"} + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + @app.get("/") async def root(): """API info""" From 5a4f21fad987ef074fdabfcb962d8dd273cabd69 Mon Sep 17 00:00:00 2001 From: unclecode Date: Thu, 9 Oct 2025 22:26:15 +0800 Subject: [PATCH 050/119] fix(marketplace): isolate api under marketplace prefix --- docs/md_v2/marketplace/admin/admin.js | 4 +- docs/md_v2/marketplace/admin/index.html | 2 +- docs/md_v2/marketplace/app-detail.js | 2 +- docs/md_v2/marketplace/backend/server.py | 64 ++++++++++--------- docs/md_v2/marketplace/frontend/app-detail.js | 2 +- .../md_v2/marketplace/frontend/marketplace.js | 2 +- docs/md_v2/marketplace/marketplace.js | 4 +- 7 files changed, 42 insertions(+), 38 deletions(-) diff --git a/docs/md_v2/marketplace/admin/admin.js b/docs/md_v2/marketplace/admin/admin.js index 7d31a826..d43dd822 100644 --- a/docs/md_v2/marketplace/admin/admin.js +++ b/docs/md_v2/marketplace/admin/admin.js @@ -30,10 +30,10 @@ const { API_BASE, API_ORIGIN } = (() => { if (origin) { const normalized = cleanOrigin(origin); - return { API_BASE: `${normalized}/api`, API_ORIGIN: normalized }; + return { API_BASE: `${normalized}/marketplace/api`, API_ORIGIN: normalized }; } - return { API_BASE: '/api', API_ORIGIN: '' }; + return { API_BASE: '/marketplace/api', API_ORIGIN: '' }; })(); const resolveAssetUrl = (path) => { diff --git a/docs/md_v2/marketplace/admin/index.html b/docs/md_v2/marketplace/admin/index.html index a9b649f2..0d302384 100644 --- a/docs/md_v2/marketplace/admin/index.html +++ b/docs/md_v2/marketplace/admin/index.html @@ -210,6 +210,6 @@
- + \ No newline at end of file diff --git a/docs/md_v2/marketplace/app-detail.js b/docs/md_v2/marketplace/app-detail.js index d1b3b559..622f9c88 100644 --- a/docs/md_v2/marketplace/app-detail.js +++ b/docs/md_v2/marketplace/app-detail.js @@ -1,5 +1,5 @@ // App Detail Page JavaScript -const API_BASE = '/api'; +const API_BASE = '/marketplace/api'; class AppDetailPage { constructor() { diff --git a/docs/md_v2/marketplace/backend/server.py b/docs/md_v2/marketplace/backend/server.py index 0f177386..a414ad80 100644 --- a/docs/md_v2/marketplace/backend/server.py +++ b/docs/md_v2/marketplace/backend/server.py @@ -1,4 +1,4 @@ -from fastapi import FastAPI, HTTPException, Query, Depends, Body, UploadFile, File, Form +from fastapi import FastAPI, HTTPException, Query, Depends, Body, UploadFile, File, Form, APIRouter from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse from fastapi.staticfiles import StaticFiles @@ -16,6 +16,7 @@ from datetime import datetime, timedelta from config import Config app = FastAPI(title="Crawl4AI Marketplace API") +router = APIRouter(prefix="/marketplace/api") # Security setup security = HTTPBearer() @@ -84,7 +85,7 @@ def to_int(value, default=0): # ============= PUBLIC ENDPOINTS ============= -@app.get("/api/apps") +@router.get("/apps") async def get_apps( category: Optional[str] = None, type: Optional[str] = None, @@ -114,7 +115,7 @@ async def get_apps( return json_response(apps) -@app.get("/api/apps/{slug}") +@router.get("/apps/{slug}") async def get_app(slug: str): """Get single app by slug""" apps = db.get_all('apps', where=f"slug = '{slug}'", limit=1) @@ -127,7 +128,7 @@ async def get_app(slug: str): return json_response(app) -@app.get("/api/articles") +@router.get("/articles") async def get_articles( category: Optional[str] = None, limit: int = Query(default=20, le=10000), @@ -146,7 +147,7 @@ async def get_articles( return json_response(articles) -@app.get("/api/articles/{slug}") +@router.get("/articles/{slug}") async def get_article(slug: str): """Get single article by slug""" articles = db.get_all('articles', where=f"slug = '{slug}'", limit=1) @@ -161,7 +162,7 @@ async def get_article(slug: str): return json_response(article) -@app.get("/api/categories") +@router.get("/categories") async def get_categories(): """Get all categories ordered by index""" categories = db.get_all('categories', limit=50) @@ -170,7 +171,7 @@ async def get_categories(): categories.sort(key=lambda x: x.get('order_index', 0)) return json_response(categories, cache_time=7200) -@app.get("/api/sponsors") +@router.get("/sponsors") async def get_sponsors(active: Optional[bool] = True): """Get sponsors, default active only""" where = f"active = {1 if active else 0}" if active is not None else None @@ -185,7 +186,7 @@ async def get_sponsors(active: Optional[bool] = True): return json_response(sponsors) -@app.get("/api/search") +@router.get("/search") async def search(q: str = Query(min_length=2)): """Search across apps and articles""" if len(q) < 2: @@ -206,7 +207,7 @@ async def search(q: str = Query(min_length=2)): return json_response(results, cache_time=1800) -@app.get("/api/stats") +@router.get("/stats") async def get_stats(): """Get marketplace statistics""" stats = { @@ -227,7 +228,7 @@ def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)): return token -@app.post("/api/admin/upload-image", dependencies=[Depends(verify_token)]) +@router.post("/admin/upload-image", dependencies=[Depends(verify_token)]) async def upload_image(file: UploadFile = File(...), folder: str = Form("sponsors")): """Upload image files for admin assets""" folder = (folder or "").strip().lower() @@ -251,7 +252,7 @@ async def upload_image(file: UploadFile = File(...), folder: str = Form("sponsor return {"url": f"/uploads/{folder}/{filename}"} -@app.post("/api/admin/login") +@router.post("/admin/login") async def admin_login(password: str = Body(..., embed=True)): """Admin login with password""" provided_hash = hashlib.sha256(password.encode()).hexdigest() @@ -272,7 +273,7 @@ async def admin_login(password: str = Body(..., embed=True)): # ============= ADMIN ENDPOINTS ============= -@app.get("/api/admin/stats", dependencies=[Depends(verify_token)]) +@router.get("/admin/stats", dependencies=[Depends(verify_token)]) async def get_admin_stats(): """Get detailed admin statistics""" stats = { @@ -292,7 +293,7 @@ async def get_admin_stats(): return stats # Apps CRUD -@app.post("/api/admin/apps", dependencies=[Depends(verify_token)]) +@router.post("/admin/apps", dependencies=[Depends(verify_token)]) async def create_app(app_data: Dict[str, Any]): """Create new app""" try: @@ -311,7 +312,7 @@ async def create_app(app_data: Dict[str, Any]): except Exception as e: raise HTTPException(status_code=400, detail=str(e)) -@app.put("/api/admin/apps/{app_id}", dependencies=[Depends(verify_token)]) +@router.put("/admin/apps/{app_id}", dependencies=[Depends(verify_token)]) async def update_app(app_id: int, app_data: Dict[str, Any]): """Update app""" try: @@ -329,7 +330,7 @@ async def update_app(app_id: int, app_data: Dict[str, Any]): except Exception as e: raise HTTPException(status_code=400, detail=str(e)) -@app.delete("/api/admin/apps/{app_id}", dependencies=[Depends(verify_token)]) +@router.delete("/admin/apps/{app_id}", dependencies=[Depends(verify_token)]) async def delete_app(app_id: int): """Delete app""" cursor = db.conn.cursor() @@ -338,7 +339,7 @@ async def delete_app(app_id: int): return {"message": "App deleted"} # Articles CRUD -@app.post("/api/admin/articles", dependencies=[Depends(verify_token)]) +@router.post("/admin/articles", dependencies=[Depends(verify_token)]) async def create_article(article_data: Dict[str, Any]): """Create new article""" try: @@ -356,7 +357,7 @@ async def create_article(article_data: Dict[str, Any]): except Exception as e: raise HTTPException(status_code=400, detail=str(e)) -@app.put("/api/admin/articles/{article_id}", dependencies=[Depends(verify_token)]) +@router.put("/admin/articles/{article_id}", dependencies=[Depends(verify_token)]) async def update_article(article_id: int, article_data: Dict[str, Any]): """Update article""" try: @@ -373,7 +374,7 @@ async def update_article(article_id: int, article_data: Dict[str, Any]): except Exception as e: raise HTTPException(status_code=400, detail=str(e)) -@app.delete("/api/admin/articles/{article_id}", dependencies=[Depends(verify_token)]) +@router.delete("/admin/articles/{article_id}", dependencies=[Depends(verify_token)]) async def delete_article(article_id: int): """Delete article""" cursor = db.conn.cursor() @@ -382,7 +383,7 @@ async def delete_article(article_id: int): return {"message": "Article deleted"} # Categories CRUD -@app.post("/api/admin/categories", dependencies=[Depends(verify_token)]) +@router.post("/admin/categories", dependencies=[Depends(verify_token)]) async def create_category(category_data: Dict[str, Any]): """Create new category""" try: @@ -399,7 +400,7 @@ async def create_category(category_data: Dict[str, Any]): except Exception as e: raise HTTPException(status_code=400, detail=str(e)) -@app.put("/api/admin/categories/{cat_id}", dependencies=[Depends(verify_token)]) +@router.put("/admin/categories/{cat_id}", dependencies=[Depends(verify_token)]) async def update_category(cat_id: int, category_data: Dict[str, Any]): """Update category""" try: @@ -417,7 +418,7 @@ async def update_category(cat_id: int, category_data: Dict[str, Any]): raise HTTPException(status_code=400, detail=str(e)) -@app.delete("/api/admin/categories/{cat_id}", dependencies=[Depends(verify_token)]) +@router.delete("/admin/categories/{cat_id}", dependencies=[Depends(verify_token)]) async def delete_category(cat_id: int): """Delete category""" try: @@ -429,7 +430,7 @@ async def delete_category(cat_id: int): raise HTTPException(status_code=400, detail=str(e)) # Sponsors CRUD -@app.post("/api/admin/sponsors", dependencies=[Depends(verify_token)]) +@router.post("/admin/sponsors", dependencies=[Depends(verify_token)]) async def create_sponsor(sponsor_data: Dict[str, Any]): """Create new sponsor""" try: @@ -443,7 +444,7 @@ async def create_sponsor(sponsor_data: Dict[str, Any]): except Exception as e: raise HTTPException(status_code=400, detail=str(e)) -@app.put("/api/admin/sponsors/{sponsor_id}", dependencies=[Depends(verify_token)]) +@router.put("/admin/sponsors/{sponsor_id}", dependencies=[Depends(verify_token)]) async def update_sponsor(sponsor_id: int, sponsor_data: Dict[str, Any]): """Update sponsor""" try: @@ -457,7 +458,7 @@ async def update_sponsor(sponsor_id: int, sponsor_data: Dict[str, Any]): raise HTTPException(status_code=400, detail=str(e)) -@app.delete("/api/admin/sponsors/{sponsor_id}", dependencies=[Depends(verify_token)]) +@router.delete("/admin/sponsors/{sponsor_id}", dependencies=[Depends(verify_token)]) async def delete_sponsor(sponsor_id: int): """Delete sponsor""" try: @@ -468,6 +469,9 @@ async def delete_sponsor(sponsor_id: int): except Exception as e: raise HTTPException(status_code=400, detail=str(e)) +app.include_router(router) + + @app.get("/") async def root(): """API info""" @@ -475,12 +479,12 @@ async def root(): "name": "Crawl4AI Marketplace API", "version": "1.0.0", "endpoints": [ - "/api/apps", - "/api/articles", - "/api/categories", - "/api/sponsors", - "/api/search?q=query", - "/api/stats" + "/marketplace/api/apps", + "/marketplace/api/articles", + "/marketplace/api/categories", + "/marketplace/api/sponsors", + "/marketplace/api/search?q=query", + "/marketplace/api/stats" ] } diff --git a/docs/md_v2/marketplace/frontend/app-detail.js b/docs/md_v2/marketplace/frontend/app-detail.js index d1b3b559..622f9c88 100644 --- a/docs/md_v2/marketplace/frontend/app-detail.js +++ b/docs/md_v2/marketplace/frontend/app-detail.js @@ -1,5 +1,5 @@ // App Detail Page JavaScript -const API_BASE = '/api'; +const API_BASE = '/marketplace/api'; class AppDetailPage { constructor() { diff --git a/docs/md_v2/marketplace/frontend/marketplace.js b/docs/md_v2/marketplace/frontend/marketplace.js index 94a401bf..df07257c 100644 --- a/docs/md_v2/marketplace/frontend/marketplace.js +++ b/docs/md_v2/marketplace/frontend/marketplace.js @@ -1,5 +1,5 @@ // Marketplace JS - Magazine Layout -const API_BASE = '/api'; +const API_BASE = '/marketplace/api'; const CACHE_TTL = 3600000; // 1 hour in ms class MarketplaceCache { diff --git a/docs/md_v2/marketplace/marketplace.js b/docs/md_v2/marketplace/marketplace.js index 84022c47..7813e3ba 100644 --- a/docs/md_v2/marketplace/marketplace.js +++ b/docs/md_v2/marketplace/marketplace.js @@ -3,9 +3,9 @@ const { API_BASE, API_ORIGIN } = (() => { const { hostname, port } = window.location; if ((hostname === 'localhost' || hostname === '127.0.0.1') && port === '8000') { const origin = 'http://127.0.0.1:8100'; - return { API_BASE: `${origin}/api`, API_ORIGIN: origin }; + return { API_BASE: `${origin}/marketplace/api`, API_ORIGIN: origin }; } - return { API_BASE: '/api', API_ORIGIN: '' }; + return { API_BASE: '/marketplace/api', API_ORIGIN: '' }; })(); const resolveAssetUrl = (path) => { From abe8a92561b008e611bcfa4c603adab4901fac3e Mon Sep 17 00:00:00 2001 From: unclecode Date: Sat, 11 Oct 2025 11:51:22 +0800 Subject: [PATCH 051/119] fix(marketplace): resolve app detail page routing and styling issues - Fixed JavaScript errors from missing HTML elements (install-code, usage-code, integration-code) - Added missing CSS classes for tabs, overview layout, sidebar, and integration content - Fixed tab navigation to display horizontally in single line - Added proper padding to tab content sections (removed from container, added to content) - Fixed tab selector from .nav-tab to .tab-btn to match HTML structure - Added sidebar styling with stats grid and metadata display - Improved responsive design with mobile-friendly tab scrolling - Fixed code block positioning for copy buttons - Removed margin from first headings to prevent extra spacing - Added null checks for DOM elements in JavaScript to prevent errors These changes resolve the routing issue where clicking on apps caused page redirects, and fix the broken layout where CSS was not properly applied to the app detail page. --- docs/md_v2/marketplace/app-detail.css | 214 +++++++++++++++++- docs/md_v2/marketplace/app-detail.html | 209 +++++++++++++++++ docs/md_v2/marketplace/app-detail.js | 44 +++- docs/md_v2/marketplace/frontend/app-detail.js | 12 +- 4 files changed, 459 insertions(+), 20 deletions(-) create mode 100644 docs/md_v2/marketplace/app-detail.html diff --git a/docs/md_v2/marketplace/app-detail.css b/docs/md_v2/marketplace/app-detail.css index 9f04c13a..590bea03 100644 --- a/docs/md_v2/marketplace/app-detail.css +++ b/docs/md_v2/marketplace/app-detail.css @@ -197,6 +197,41 @@ } /* Navigation Tabs */ +.tabs { + display: flex; + flex-direction: row; + gap: 0; + border-bottom: 2px solid var(--border-color); + margin-bottom: 0; + background: var(--bg-tertiary); +} + +.tab-btn { + padding: 1rem 2rem; + background: transparent; + border: none; + border-bottom: 3px solid transparent; + color: var(--text-secondary); + cursor: pointer; + transition: all 0.2s; + font-family: inherit; + font-size: 0.95rem; + margin-bottom: -2px; + white-space: nowrap; + font-weight: 500; +} + +.tab-btn:hover { + color: var(--primary-cyan); + background: rgba(80, 255, 255, 0.05); +} + +.tab-btn.active { + color: var(--primary-cyan); + border-bottom-color: var(--primary-cyan); + background: var(--bg-secondary); +} + .app-nav { max-width: 1800px; margin: 2rem auto 0; @@ -228,34 +263,167 @@ border-bottom-color: var(--primary-cyan); } -/* Content Sections */ -.app-content { +/* Main Content Wrapper */ +.app-main { max-width: 1800px; margin: 2rem auto; padding: 0 2rem; } +/* Content Sections */ +.app-content { + background: var(--bg-secondary); + border: 1px solid var(--border-color); + padding: 0; +} + .tab-content { display: none; + padding: 2rem; } .tab-content.active { display: block; } -.docs-content { - max-width: 1200px; - padding: 2rem; +/* Overview Layout */ +.overview-columns { + display: grid; + grid-template-columns: 2fr 1fr; + gap: 2rem; +} + +.overview-main h2, .overview-main h3 { + color: var(--primary-cyan); + margin-top: 2rem; + margin-bottom: 1rem; +} + +.overview-main h2:first-child { + margin-top: 0; +} + +.overview-main h2 { + font-size: 1.8rem; + border-bottom: 2px solid var(--border-color); + padding-bottom: 0.5rem; +} + +.overview-main h3 { + font-size: 1.3rem; +} + +.features-list { + list-style: none; + padding: 0; +} + +.features-list li { + padding: 0.5rem 0; + padding-left: 1.5rem; + position: relative; + color: var(--text-secondary); +} + +.features-list li:before { + content: "▸"; + position: absolute; + left: 0; + color: var(--primary-cyan); +} + +.use-cases p { + color: var(--text-secondary); + line-height: 1.6; +} + +/* Sidebar */ +.sidebar { + display: flex; + flex-direction: column; + gap: 1rem; +} + +.sidebar-card { background: var(--bg-secondary); border: 1px solid var(--border-color); + padding: 1.5rem; +} + +.sidebar-card h3 { + font-size: 1.1rem; + color: var(--primary-cyan); + margin: 0 0 1rem 0; + border-bottom: 1px solid var(--border-color); + padding-bottom: 0.5rem; +} + +.stats-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 1rem; +} + +.stats-grid > div { + text-align: center; +} + +.metadata { + margin: 0; +} + +.metadata div { + display: flex; + justify-content: space-between; + padding: 0.75rem 0; + border-bottom: 1px solid var(--border-color); +} + +.metadata dt { + color: var(--text-tertiary); + font-weight: normal; +} + +.metadata dd { + color: var(--text-primary); + margin: 0; + font-weight: 600; +} + +.sidebar-card p { + color: var(--text-secondary); + margin: 0; +} + +/* Integration Content */ +.integration-content { + max-width: 100%; +} + +.integration-content h2 { + font-size: 1.8rem; + color: var(--primary-cyan); + margin: 0 0 2rem 0; + padding-bottom: 0.5rem; + border-bottom: 2px solid var(--border-color); +} + +.integration-content h3 { + font-size: 1.3rem; + color: var(--text-primary); + margin: 2rem 0 1rem; +} + +.docs-content { + max-width: 100%; } .docs-content h2 { font-size: 1.8rem; color: var(--primary-cyan); - margin-bottom: 1rem; + margin: 0 0 1.5rem 0; padding-bottom: 0.5rem; - border-bottom: 1px solid var(--border-color); + border-bottom: 2px solid var(--border-color); } .docs-content h3 { @@ -290,6 +458,7 @@ border: 1px solid var(--border-color); margin: 1rem 0; overflow: hidden; + position: relative; } .code-header { @@ -308,18 +477,23 @@ } .copy-btn { - padding: 0.25rem 0.5rem; - background: transparent; + position: absolute; + top: 0.5rem; + right: 0.5rem; + padding: 0.4rem 0.8rem; + background: var(--bg-tertiary); border: 1px solid var(--border-color); color: var(--text-secondary); cursor: pointer; font-size: 0.75rem; transition: all 0.2s; + z-index: 10; } .copy-btn:hover { border-color: var(--primary-cyan); color: var(--primary-cyan); + background: var(--bg-secondary); } .code-block pre { @@ -435,6 +609,10 @@ .app-stats { justify-content: space-around; } + + .overview-columns { + grid-template-columns: 1fr; + } } @media (max-width: 768px) { @@ -446,6 +624,16 @@ flex-direction: column; } + .tabs { + overflow-x: auto; + -webkit-overflow-scrolling: touch; + } + + .tab-btn { + padding: 0.75rem 1.5rem; + font-size: 0.875rem; + } + .app-nav { overflow-x: auto; gap: 0; @@ -459,4 +647,12 @@ .support-grid { grid-template-columns: 1fr; } + + .tab-content { + padding: 1rem; + } + + .app-main { + padding: 0 1rem; + } } \ No newline at end of file diff --git a/docs/md_v2/marketplace/app-detail.html b/docs/md_v2/marketplace/app-detail.html new file mode 100644 index 00000000..ef1138a8 --- /dev/null +++ b/docs/md_v2/marketplace/app-detail.html @@ -0,0 +1,209 @@ + + + + + + App Details - Crawl4AI Marketplace + + + + +
+ +
+
+
+
+ +

+ [ + Marketplace + ] +

+
+
+
+ ← Back to Marketplace +
+
+
+ + +
+
+
+ +
+
+
+ Open Source + + +
+

App Name

+

App description goes here

+ +
+
+ ★★★★★ + Rating +
+
+ 0 + Downloads +
+
+ Category + Category +
+
+ +
+ Visit Website + View GitHub + +
+
+
+
+ + +
+
+
+ + + + +
+ +
+
+
+

Overview

+
Overview content goes here.
+ +

Key Features

+
    +
  • Feature 1
    • +
  • Feature 2
    • +
  • Feature 3
    • +
+ +

Use Cases

+
+

Describe how this app can help your workflow.

+
+
+ + +
+
+ +
+
+

Integration Guide

+ +

Installation

+
+ 
# Installation instructions will appear here
+
+ +

Basic Usage

+
+ 
# Usage example will appear here
+
+ +

Complete Integration Example

+
+ + 
# Complete integration guide will appear here
+
+
+
+ +
+
+

Documentation

+
+

Documentation coming soon.

+
+
+
+ +
+
+

Support

+
+
+

📧 Contact

+

contact@example.com

+
+
+

🐛 Report Issues

+

Found a bug? Report it on GitHub Issues.

+
+
+

💬 Community

+

Join our Discord for help and discussions.

+
+
+
+
+
+ +
+ + +
+

Related Apps

+ +
+
+ + + + diff --git a/docs/md_v2/marketplace/app-detail.js b/docs/md_v2/marketplace/app-detail.js index 622f9c88..f470bf51 100644 --- a/docs/md_v2/marketplace/app-detail.js +++ b/docs/md_v2/marketplace/app-detail.js @@ -1,5 +1,15 @@ // App Detail Page JavaScript -const API_BASE = '/marketplace/api'; +const { API_BASE, API_ORIGIN } = (() => { + const { hostname, port, protocol } = window.location; + const isLocalHost = ['localhost', '127.0.0.1', '0.0.0.0'].includes(hostname); + + if (isLocalHost && port && port !== '8100') { + const origin = `${protocol}//127.0.0.1:8100`; + return { API_BASE: `${origin}/marketplace/api`, API_ORIGIN: origin }; + } + + return { API_BASE: '/marketplace/api', API_ORIGIN: '' }; +})(); class AppDetailPage { constructor() { @@ -70,7 +80,6 @@ class AppDetailPage { document.getElementById('app-description').textContent = this.appData.description; document.getElementById('app-type').textContent = this.appData.type || 'Open Source'; document.getElementById('app-category').textContent = this.appData.category; - document.getElementById('app-pricing').textContent = this.appData.pricing || 'Free'; // Badges if (this.appData.featured) { @@ -105,6 +114,15 @@ class AppDetailPage { // Contact document.getElementById('app-contact').textContent = this.appData.contact_email || 'Not available'; + // Sidebar info + document.getElementById('sidebar-downloads').textContent = this.formatNumber(this.appData.downloads || 0); + document.getElementById('sidebar-rating').textContent = (this.appData.rating || 0).toFixed(1); + document.getElementById('sidebar-category').textContent = this.appData.category || '-'; + document.getElementById('sidebar-type').textContent = this.appData.type || '-'; + document.getElementById('sidebar-status').textContent = this.appData.status || 'Active'; + document.getElementById('sidebar-pricing').textContent = this.appData.pricing || 'Free'; + document.getElementById('sidebar-contact').textContent = this.appData.contact_email || 'contact@example.com'; + // Integration guide this.renderIntegrationGuide(); } @@ -112,24 +130,27 @@ class AppDetailPage { renderIntegrationGuide() { // Installation code const installCode = document.getElementById('install-code'); - if (this.appData.type === 'Open Source' && this.appData.github_url) { - installCode.textContent = `# Clone from GitHub + if (installCode) { + if (this.appData.type === 'Open Source' && this.appData.github_url) { + installCode.textContent = `# Clone from GitHub git clone ${this.appData.github_url} # Install dependencies pip install -r requirements.txt`; - } else if (this.appData.name.toLowerCase().includes('api')) { - installCode.textContent = `# Install via pip + } else if (this.appData.name.toLowerCase().includes('api')) { + installCode.textContent = `# Install via pip pip install ${this.appData.slug} # Or install from source pip install git+${this.appData.github_url || 'https://github.com/example/repo'}`; + } } // Usage code - customize based on category const usageCode = document.getElementById('usage-code'); - if (this.appData.category === 'Browser Automation') { - usageCode.textContent = `from crawl4ai import AsyncWebCrawler + if (usageCode) { + if (this.appData.category === 'Browser Automation') { + usageCode.textContent = `from crawl4ai import AsyncWebCrawler from ${this.appData.slug.replace(/-/g, '_')} import ${this.appData.name.replace(/\s+/g, '')} async def main(): @@ -178,11 +199,13 @@ async with AsyncWebCrawler() as crawler: extraction_strategy=strategy ) print(result.extracted_content)`; + } } // Integration example const integrationCode = document.getElementById('integration-code'); - integrationCode.textContent = this.appData.integration_guide || + if (integrationCode) { + integrationCode.textContent = this.appData.integration_guide || `# Complete ${this.appData.name} Integration Example from crawl4ai import AsyncWebCrawler @@ -237,6 +260,7 @@ async def crawl_with_${this.appData.slug.replace(/-/g, '_')}(): if __name__ == "__main__": import asyncio asyncio.run(crawl_with_${this.appData.slug.replace(/-/g, '_')}())`; + } } formatNumber(num) { @@ -250,7 +274,7 @@ if __name__ == "__main__": setupEventListeners() { // Tab switching - const tabs = document.querySelectorAll('.nav-tab'); + const tabs = document.querySelectorAll('.tab-btn'); tabs.forEach(tab => { tab.addEventListener('click', () => { // Update active tab diff --git a/docs/md_v2/marketplace/frontend/app-detail.js b/docs/md_v2/marketplace/frontend/app-detail.js index 622f9c88..5bc86d2b 100644 --- a/docs/md_v2/marketplace/frontend/app-detail.js +++ b/docs/md_v2/marketplace/frontend/app-detail.js @@ -1,5 +1,15 @@ // App Detail Page JavaScript -const API_BASE = '/marketplace/api'; +const { API_BASE, API_ORIGIN } = (() => { + const { hostname, port, protocol } = window.location; + const isLocalHost = ['localhost', '127.0.0.1', '0.0.0.0'].includes(hostname); + + if (isLocalHost && port && port !== '8100') { + const origin = `${protocol}//127.0.0.1:8100`; + return { API_BASE: `${origin}/marketplace/api`, API_ORIGIN: origin }; + } + + return { API_BASE: '/marketplace/api', API_ORIGIN: '' }; +})(); class AppDetailPage { constructor() { From 216019f29afb7cf118535093af7f2fd83b9fa5d4 Mon Sep 17 00:00:00 2001 From: unclecode Date: Sat, 11 Oct 2025 12:52:04 +0800 Subject: [PATCH 052/119] fix(marketplace): prevent hero image overflow and secondary card stretching - Fixed hero image to 200px height with min/max constraints - Added object-fit: cover to hero-image img elements - Changed secondary-featured align-items from stretch to flex-start - Fixed secondary-card height to 118px (no flex: 1 stretching) - Updated responsive grid layouts for wider screens - Added flex: 1 to hero-content for better content distribution These changes ensure a rigid, predictable layout that prevents: 1. Large images from pushing text content down 2. Single secondary cards from stretching to fill entire height --- docs/md_v2/marketplace/marketplace.css | 34 +++++++++++++++++++++----- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/docs/md_v2/marketplace/marketplace.css b/docs/md_v2/marketplace/marketplace.css index a95c4a41..57f54d22 100644 --- a/docs/md_v2/marketplace/marketplace.css +++ b/docs/md_v2/marketplace/marketplace.css @@ -231,7 +231,9 @@ a:hover { .hero-image { width: 100%; - height: 240px; + height: 200px; + min-height: 200px; + max-height: 200px; background: linear-gradient(135deg, rgba(80, 255, 255, 0.1), rgba(243, 128, 245, 0.05)); background-size: cover; background-position: center; @@ -243,6 +245,14 @@ a:hover { flex-shrink: 0; position: relative; filter: brightness(1.1) contrast(1.1); + overflow: hidden; +} + +.hero-image img { + width: 100%; + height: 100%; + object-fit: cover; + object-position: center; } .hero-image::after { @@ -257,6 +267,10 @@ a:hover { .hero-content { padding: 1.5rem; + flex: 1; + display: flex; + flex-direction: column; + justify-content: space-between; } .hero-badge { @@ -301,9 +315,9 @@ a:hover { /* Secondary Featured */ .secondary-featured { grid-column: 1 / -1; - height: 380px; + min-height: 380px; display: flex; - align-items: stretch; + align-items: flex-start; } .featured-secondary-cards { @@ -311,7 +325,7 @@ a:hover { display: flex; flex-direction: column; gap: 0.75rem; - justify-content: space-between; + align-items: stretch; } .secondary-card { @@ -321,8 +335,10 @@ a:hover { transition: all 0.3s ease; display: flex; overflow: hidden; - height: calc((380px - 1.5rem) / 3); - flex: 1; + height: 118px; + min-height: 118px; + max-height: 118px; + flex-shrink: 0; box-shadow: 0 2px 10px rgba(0, 0, 0, 0.3); } @@ -875,10 +891,13 @@ a:hover { .secondary-featured { grid-column: 3 / 5; grid-row: 1; + min-height: auto; } .featured-secondary-cards { + display: grid; grid-template-columns: repeat(2, 1fr); + flex-direction: unset; } .main-content { @@ -906,10 +925,13 @@ a:hover { .secondary-featured { grid-column: 3 / 6; + min-height: auto; } .featured-secondary-cards { + display: grid; grid-template-columns: repeat(3, 1fr); + flex-direction: unset; } .sponsored-section { From a3f057e19fc60245ef16c9b8fb1639c7b1555fb2 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 13 Oct 2025 12:34:08 +0800 Subject: [PATCH 053/119] feat: Add hooks utility for function-based hooks with Docker client integration. ref #1377 Add hooks_to_string() utility function that converts Python function objects to string representations for the Docker API, enabling developers to write hooks as regular Python functions instead of strings. Core Changes: - New hooks_to_string() utility in crawl4ai/utils.py using inspect.getsource() - Docker client now accepts both function objects and strings for hooks - Automatic detection and conversion in Crawl4aiDockerClient._prepare_request() - New hooks and hooks_timeout parameters in client.crawl() method Documentation: - Docker client examples with function-based hooks (docs/examples/docker_client_hooks_example.py) - Updated main Docker deployment guide with comprehensive hooks section - Added unit tests for hooks utility (tests/docker/test_hooks_utility.py) --- crawl4ai/__init__.py | 4 +- crawl4ai/docker_client.py | 75 ++- crawl4ai/utils.py | 51 +- docs/examples/docker_client_hooks_example.py | 522 +++++++++++++++++++ docs/md_v2/core/docker-deployment.md | 397 ++++++++++++-- tests/docker/test_hooks_utility.py | 193 +++++++ 6 files changed, 1198 insertions(+), 44 deletions(-) create mode 100644 docs/examples/docker_client_hooks_example.py create mode 100644 tests/docker/test_hooks_utility.py diff --git a/crawl4ai/__init__.py b/crawl4ai/__init__.py index 6917f27e..8f1fdef4 100644 --- a/crawl4ai/__init__.py +++ b/crawl4ai/__init__.py @@ -103,7 +103,8 @@ from .browser_adapter import ( from .utils import ( start_colab_display_server, - setup_colab_environment + setup_colab_environment, + hooks_to_string ) __all__ = [ @@ -183,6 +184,7 @@ __all__ = [ "ProxyConfig", "start_colab_display_server", "setup_colab_environment", + "hooks_to_string", # C4A Script additions "c4a_compile", "c4a_validate", diff --git a/crawl4ai/docker_client.py b/crawl4ai/docker_client.py index 4e33431f..969fee7c 100644 --- a/crawl4ai/docker_client.py +++ b/crawl4ai/docker_client.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Union, AsyncGenerator, Dict, Any +from typing import List, Optional, Union, AsyncGenerator, Dict, Any, Callable import httpx import json from urllib.parse import urljoin @@ -7,6 +7,7 @@ import asyncio from .async_configs import BrowserConfig, CrawlerRunConfig from .models import CrawlResult from .async_logger import AsyncLogger, LogLevel +from .utils import hooks_to_string class Crawl4aiClientError(Exception): @@ -70,17 +71,41 @@ class Crawl4aiDockerClient: self.logger.error(f"Server unreachable: {str(e)}", tag="ERROR") raise ConnectionError(f"Cannot connect to server: {str(e)}") - def _prepare_request(self, urls: List[str], browser_config: Optional[BrowserConfig] = None, - crawler_config: Optional[CrawlerRunConfig] = None) -> Dict[str, Any]: + def _prepare_request( + self, + urls: List[str], + browser_config: Optional[BrowserConfig] = None, + crawler_config: Optional[CrawlerRunConfig] = None, + hooks: Optional[Union[Dict[str, Callable], Dict[str, str]]] = None, + hooks_timeout: int = 30 + ) -> Dict[str, Any]: """Prepare request data from configs.""" if self._token: self._http_client.headers["Authorization"] = f"Bearer {self._token}" - return { + + request_data = { "urls": urls, "browser_config": browser_config.dump() if browser_config else {}, "crawler_config": crawler_config.dump() if crawler_config else {} } + # Handle hooks if provided + if hooks: + # Check if hooks are already strings or need conversion + if any(callable(v) for v in hooks.values()): + # Convert function objects to strings + hooks_code = hooks_to_string(hooks) + else: + # Already in string format + hooks_code = hooks + + request_data["hooks"] = { + "code": hooks_code, + "timeout": hooks_timeout + } + + return request_data + async def _request(self, method: str, endpoint: str, **kwargs) -> httpx.Response: """Make an HTTP request with error handling.""" url = urljoin(self.base_url, endpoint) @@ -102,16 +127,42 @@ class Crawl4aiDockerClient: self, urls: List[str], browser_config: Optional[BrowserConfig] = None, - crawler_config: Optional[CrawlerRunConfig] = None + crawler_config: Optional[CrawlerRunConfig] = None, + hooks: Optional[Union[Dict[str, Callable], Dict[str, str]]] = None, + hooks_timeout: int = 30 ) -> Union[CrawlResult, List[CrawlResult], AsyncGenerator[CrawlResult, None]]: - """Execute a crawl operation.""" + """ + Execute a crawl operation. + + Args: + urls: List of URLs to crawl + browser_config: Browser configuration + crawler_config: Crawler configuration + hooks: Optional hooks - can be either: + - Dict[str, Callable]: Function objects that will be converted to strings + - Dict[str, str]: Already stringified hook code + hooks_timeout: Timeout in seconds for each hook execution (1-120) + + Returns: + Single CrawlResult, list of results, or async generator for streaming + + Example with function hooks: + >>> async def my_hook(page, context, **kwargs): + ... await page.set_viewport_size({"width": 1920, "height": 1080}) + ... return page + >>> + >>> result = await client.crawl( + ... ["https://example.com"], + ... hooks={"on_page_context_created": my_hook} + ... ) + """ await self._check_server() - - data = self._prepare_request(urls, browser_config, crawler_config) + + data = self._prepare_request(urls, browser_config, crawler_config, hooks, hooks_timeout) is_streaming = crawler_config and crawler_config.stream - + self.logger.info(f"Crawling {len(urls)} URLs {'(streaming)' if is_streaming else ''}", tag="CRAWL") - + if is_streaming: async def stream_results() -> AsyncGenerator[CrawlResult, None]: async with self._http_client.stream("POST", f"{self.base_url}/crawl/stream", json=data) as response: @@ -128,12 +179,12 @@ class Crawl4aiDockerClient: else: yield CrawlResult(**result) return stream_results() - + response = await self._request("POST", "/crawl", json=data) result_data = response.json() if not result_data.get("success", False): raise RequestError(f"Crawl failed: {result_data.get('msg', 'Unknown error')}") - + results = [CrawlResult(**r) for r in result_data.get("results", [])] self.logger.success(f"Crawl completed with {len(results)} results", tag="CRAWL") return results[0] if len(results) == 1 else results diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py index 046351e7..bbd7ffa2 100644 --- a/crawl4ai/utils.py +++ b/crawl4ai/utils.py @@ -47,6 +47,7 @@ from urllib.parse import ( urljoin, urlparse, urlunparse, parse_qsl, urlencode, quote, unquote ) +import inspect # Monkey patch to fix wildcard handling in urllib.robotparser @@ -3529,4 +3530,52 @@ def get_memory_stats() -> Tuple[float, float, float]: available_gb = get_true_available_memory_gb() used_percent = get_true_memory_usage_percent() - return used_percent, available_gb, total_gb \ No newline at end of file + return used_percent, available_gb, total_gb + + +# Hook utilities for Docker API +def hooks_to_string(hooks: Dict[str, Callable]) -> Dict[str, str]: + """ + Convert hook function objects to string representations for Docker API. + + This utility simplifies the process of using hooks with the Docker API by converting + Python function objects into the string format required by the API. + + Args: + hooks: Dictionary mapping hook point names to Python function objects. + Functions should be async and follow hook signature requirements. + + Returns: + Dictionary mapping hook point names to string representations of the functions. + + Example: + >>> async def my_hook(page, context, **kwargs): + ... await page.set_viewport_size({"width": 1920, "height": 1080}) + ... return page + >>> + >>> hooks_dict = {"on_page_context_created": my_hook} + >>> api_hooks = hooks_to_string(hooks_dict) + >>> # api_hooks is now ready to use with Docker API + + Raises: + ValueError: If a hook is not callable or source cannot be extracted + """ + result = {} + + for hook_name, hook_func in hooks.items(): + if not callable(hook_func): + raise ValueError(f"Hook '{hook_name}' must be a callable function, got {type(hook_func)}") + + try: + # Get the source code of the function + source = inspect.getsource(hook_func) + # Remove any leading indentation to get clean source + source = textwrap.dedent(source) + result[hook_name] = source + except (OSError, TypeError) as e: + raise ValueError( + f"Cannot extract source code for hook '{hook_name}'. " + f"Make sure the function is defined in a file (not interactively). Error: {e}" + ) + + return result diff --git a/docs/examples/docker_client_hooks_example.py b/docs/examples/docker_client_hooks_example.py new file mode 100644 index 00000000..1aa27fdc --- /dev/null +++ b/docs/examples/docker_client_hooks_example.py @@ -0,0 +1,522 @@ +#!/usr/bin/env python3 +""" +Comprehensive hooks examples using Docker Client with function objects. + +This approach is recommended because: +- Write hooks as regular Python functions +- Full IDE support (autocomplete, type checking) +- Automatic conversion to API format +- Reusable and testable code +- Clean, readable syntax +""" + +import asyncio +from crawl4ai import Crawl4aiDockerClient + +# API_BASE_URL = "http://localhost:11235" +API_BASE_URL = "http://localhost:11234" + + +# ============================================================================ +# Hook Function Definitions +# ============================================================================ + +# --- All Hooks Demo --- +async def browser_created_hook(browser, **kwargs): + """Called after browser is created""" + print("[HOOK] Browser created and ready") + return browser + + +async def page_context_hook(page, context, **kwargs): + """Setup page environment""" + print("[HOOK] Setting up page environment") + + # Set viewport + await page.set_viewport_size({"width": 1920, "height": 1080}) + + # Add cookies + await context.add_cookies([{ + "name": "test_session", + "value": "abc123xyz", + "domain": ".httpbin.org", + "path": "/" + }]) + + # Block resources + await context.route("**/*.{png,jpg,jpeg,gif}", lambda route: route.abort()) + await context.route("**/analytics/*", lambda route: route.abort()) + + print("[HOOK] Environment configured") + return page + + +async def user_agent_hook(page, context, user_agent, **kwargs): + """Called when user agent is updated""" + print(f"[HOOK] User agent: {user_agent[:50]}...") + return page + + +async def before_goto_hook(page, context, url, **kwargs): + """Called before navigating to URL""" + print(f"[HOOK] Navigating to: {url}") + + await page.set_extra_http_headers({ + "X-Custom-Header": "crawl4ai-test", + "Accept-Language": "en-US" + }) + + return page + + +async def after_goto_hook(page, context, url, response, **kwargs): + """Called after page loads""" + print(f"[HOOK] Page loaded: {url}") + + await page.wait_for_timeout(1000) + + try: + await page.wait_for_selector("body", timeout=2000) + print("[HOOK] Body element ready") + except: + print("[HOOK] Timeout, continuing") + + return page + + +async def execution_started_hook(page, context, **kwargs): + """Called when custom JS execution starts""" + print("[HOOK] JS execution started") + await page.evaluate("console.log('[HOOK] Custom JS');") + return page + + +async def before_retrieve_hook(page, context, **kwargs): + """Called before retrieving HTML""" + print("[HOOK] Preparing HTML retrieval") + + # Scroll for lazy content + await page.evaluate("window.scrollTo(0, document.body.scrollHeight);") + await page.wait_for_timeout(500) + await page.evaluate("window.scrollTo(0, 0);") + + print("[HOOK] Scrolling complete") + return page + + +async def before_return_hook(page, context, html, **kwargs): + """Called before returning HTML""" + print(f"[HOOK] HTML ready: {len(html)} chars") + + metrics = await page.evaluate('''() => ({ + images: document.images.length, + links: document.links.length, + scripts: document.scripts.length + })''') + + print(f"[HOOK] Metrics - Images: {metrics['images']}, Links: {metrics['links']}") + return page + + +# --- Authentication Hooks --- +async def auth_context_hook(page, context, **kwargs): + """Setup authentication context""" + print("[HOOK] Setting up authentication") + + # Add auth cookies + await context.add_cookies([{ + "name": "auth_token", + "value": "fake_jwt_token", + "domain": ".httpbin.org", + "path": "/", + "httpOnly": True + }]) + + # Set localStorage + await page.evaluate(''' + localStorage.setItem('user_id', '12345'); + localStorage.setItem('auth_time', new Date().toISOString()); + ''') + + print("[HOOK] Auth context ready") + return page + + +async def auth_headers_hook(page, context, url, **kwargs): + """Add authentication headers""" + print(f"[HOOK] Adding auth headers for {url}") + + import base64 + credentials = base64.b64encode(b"user:passwd").decode('ascii') + + await page.set_extra_http_headers({ + 'Authorization': f'Basic {credentials}', + 'X-API-Key': 'test-key-123' + }) + + return page + + +# --- Performance Optimization Hooks --- +async def performance_hook(page, context, **kwargs): + """Optimize page for performance""" + print("[HOOK] Optimizing for performance") + + # Block resource-heavy content + await context.route("**/*.{png,jpg,jpeg,gif,webp,svg}", lambda r: r.abort()) + await context.route("**/*.{woff,woff2,ttf}", lambda r: r.abort()) + await context.route("**/*.{mp4,webm,ogg}", lambda r: r.abort()) + await context.route("**/googletagmanager.com/*", lambda r: r.abort()) + await context.route("**/google-analytics.com/*", lambda r: r.abort()) + await context.route("**/facebook.com/*", lambda r: r.abort()) + + # Disable animations + await page.add_style_tag(content=''' + *, *::before, *::after { + animation-duration: 0s !important; + transition-duration: 0s !important; + } + ''') + + print("[HOOK] Optimizations applied") + return page + + +async def cleanup_hook(page, context, **kwargs): + """Clean page before extraction""" + print("[HOOK] Cleaning page") + + await page.evaluate('''() => { + const selectors = [ + '.ad', '.ads', '.advertisement', + '.popup', '.modal', '.overlay', + '.cookie-banner', '.newsletter' + ]; + + selectors.forEach(sel => { + document.querySelectorAll(sel).forEach(el => el.remove()); + }); + + document.querySelectorAll('script, style').forEach(el => el.remove()); + }''') + + print("[HOOK] Page cleaned") + return page + + +# --- Content Extraction Hooks --- +async def wait_dynamic_content_hook(page, context, url, response, **kwargs): + """Wait for dynamic content to load""" + print(f"[HOOK] Waiting for dynamic content on {url}") + + await page.wait_for_timeout(2000) + + # Click "Load More" if exists + try: + load_more = await page.query_selector('[class*="load-more"], button:has-text("Load More")') + if load_more: + await load_more.click() + await page.wait_for_timeout(1000) + print("[HOOK] Clicked 'Load More'") + except: + pass + + return page + + +async def extract_metadata_hook(page, context, **kwargs): + """Extract page metadata""" + print("[HOOK] Extracting metadata") + + metadata = await page.evaluate('''() => { + const getMeta = (name) => { + const el = document.querySelector(`meta[name="${name}"], meta[property="${name}"]`); + return el ? el.getAttribute('content') : null; + }; + + return { + title: document.title, + description: getMeta('description'), + author: getMeta('author'), + keywords: getMeta('keywords'), + }; + }''') + + print(f"[HOOK] Metadata: {metadata}") + + # Infinite scroll + for i in range(3): + await page.evaluate("window.scrollTo(0, document.body.scrollHeight);") + await page.wait_for_timeout(1000) + print(f"[HOOK] Scroll {i+1}/3") + + return page + + +# --- Multi-URL Hooks --- +async def url_specific_hook(page, context, url, **kwargs): + """Apply URL-specific logic""" + print(f"[HOOK] Processing URL: {url}") + + # URL-specific headers + if 'html' in url: + await page.set_extra_http_headers({"X-Type": "HTML"}) + elif 'json' in url: + await page.set_extra_http_headers({"X-Type": "JSON"}) + + return page + + +async def track_progress_hook(page, context, url, response, **kwargs): + """Track crawl progress""" + status = response.status if response else 'unknown' + print(f"[HOOK] Loaded {url} - Status: {status}") + return page + + +# ============================================================================ +# Test Functions +# ============================================================================ + +async def test_all_hooks_comprehensive(): + """Test all 8 hook types""" + print("=" * 70) + print("Test 1: All Hooks Comprehensive Demo (Docker Client)") + print("=" * 70) + + async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client: + print("\nCrawling with all 8 hooks...") + + # Define hooks with function objects + hooks = { + "on_browser_created": browser_created_hook, + "on_page_context_created": page_context_hook, + "on_user_agent_updated": user_agent_hook, + "before_goto": before_goto_hook, + "after_goto": after_goto_hook, + "on_execution_started": execution_started_hook, + "before_retrieve_html": before_retrieve_hook, + "before_return_html": before_return_hook + } + + result = await client.crawl( + ["https://httpbin.org/html"], + hooks=hooks, + hooks_timeout=30 + ) + + print("\n✅ Success!") + print(f" URL: {result.url}") + print(f" Success: {result.success}") + print(f" HTML: {len(result.html)} chars") + + +async def test_authentication_workflow(): + """Test authentication with hooks""" + print("\n" + "=" * 70) + print("Test 2: Authentication Workflow (Docker Client)") + print("=" * 70) + + async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client: + print("\nTesting authentication...") + + hooks = { + "on_page_context_created": auth_context_hook, + "before_goto": auth_headers_hook + } + + result = await client.crawl( + ["https://httpbin.org/basic-auth/user/passwd"], + hooks=hooks, + hooks_timeout=15 + ) + + print("\n✅ Authentication completed") + + if result.success: + if '"authenticated"' in result.html and 'true' in result.html: + print(" ✅ Basic auth successful!") + else: + print(" ⚠️ Auth status unclear") + else: + print(f" ❌ Failed: {result.error_message}") + + +async def test_performance_optimization(): + """Test performance optimization""" + print("\n" + "=" * 70) + print("Test 3: Performance Optimization (Docker Client)") + print("=" * 70) + + async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client: + print("\nTesting performance hooks...") + + hooks = { + "on_page_context_created": performance_hook, + "before_retrieve_html": cleanup_hook + } + + result = await client.crawl( + ["https://httpbin.org/html"], + hooks=hooks, + hooks_timeout=10 + ) + + print("\n✅ Optimization completed") + print(f" HTML size: {len(result.html):,} chars") + print(" Resources blocked, ads removed") + + +async def test_content_extraction(): + """Test content extraction""" + print("\n" + "=" * 70) + print("Test 4: Content Extraction (Docker Client)") + print("=" * 70) + + async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client: + print("\nTesting extraction hooks...") + + hooks = { + "after_goto": wait_dynamic_content_hook, + "before_retrieve_html": extract_metadata_hook + } + + result = await client.crawl( + ["https://www.kidocode.com/"], + hooks=hooks, + hooks_timeout=20 + ) + + print("\n✅ Extraction completed") + print(f" URL: {result.url}") + print(f" Success: {result.success}") + print(f" Metadata: {result.metadata}") + + +async def test_multi_url_crawl(): + """Test hooks with multiple URLs""" + print("\n" + "=" * 70) + print("Test 5: Multi-URL Crawl (Docker Client)") + print("=" * 70) + + async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client: + print("\nCrawling multiple URLs...") + + hooks = { + "before_goto": url_specific_hook, + "after_goto": track_progress_hook + } + + results = await client.crawl( + [ + "https://httpbin.org/html", + "https://httpbin.org/json", + "https://httpbin.org/xml" + ], + hooks=hooks, + hooks_timeout=15 + ) + + print("\n✅ Multi-URL crawl completed") + print(f"\n Crawled {len(results)} URLs:") + for i, result in enumerate(results, 1): + status = "✅" if result.success else "❌" + print(f" {status} {i}. {result.url}") + + +async def test_reusable_hook_library(): + """Test using reusable hook library""" + print("\n" + "=" * 70) + print("Test 6: Reusable Hook Library (Docker Client)") + print("=" * 70) + + # Create a library of reusable hooks + class HookLibrary: + @staticmethod + async def block_images(page, context, **kwargs): + """Block all images""" + await context.route("**/*.{png,jpg,jpeg,gif}", lambda r: r.abort()) + print("[LIBRARY] Images blocked") + return page + + @staticmethod + async def block_analytics(page, context, **kwargs): + """Block analytics""" + await context.route("**/analytics/*", lambda r: r.abort()) + await context.route("**/google-analytics.com/*", lambda r: r.abort()) + print("[LIBRARY] Analytics blocked") + return page + + @staticmethod + async def scroll_infinite(page, context, **kwargs): + """Handle infinite scroll""" + for i in range(5): + prev = await page.evaluate("document.body.scrollHeight") + await page.evaluate("window.scrollTo(0, document.body.scrollHeight);") + await page.wait_for_timeout(1000) + curr = await page.evaluate("document.body.scrollHeight") + if curr == prev: + break + print("[LIBRARY] Infinite scroll complete") + return page + + async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client: + print("\nUsing hook library...") + + hooks = { + "on_page_context_created": HookLibrary.block_images, + "before_retrieve_html": HookLibrary.scroll_infinite + } + + result = await client.crawl( + ["https://www.kidocode.com/"], + hooks=hooks, + hooks_timeout=20 + ) + + print("\n✅ Library hooks completed") + print(f" Success: {result.success}") + + +# ============================================================================ +# Main +# ============================================================================ + +async def main(): + """Run all Docker client hook examples""" + print("🔧 Crawl4AI Docker Client - Hooks Examples (Function-Based)") + print("Using Python function objects with automatic conversion") + print("=" * 70) + + tests = [ + ("All Hooks Demo", test_all_hooks_comprehensive), + ("Authentication", test_authentication_workflow), + ("Performance", test_performance_optimization), + ("Extraction", test_content_extraction), + ("Multi-URL", test_multi_url_crawl), + ("Hook Library", test_reusable_hook_library) + ] + + for i, (name, test_func) in enumerate(tests, 1): + try: + await test_func() + print(f"\n✅ Test {i}/{len(tests)}: {name} completed\n") + except Exception as e: + print(f"\n❌ Test {i}/{len(tests)}: {name} failed: {e}\n") + import traceback + traceback.print_exc() + + print("=" * 70) + print("🎉 All Docker client hook examples completed!") + print("\n💡 Key Benefits of Function-Based Hooks:") + print(" • Write as regular Python functions") + print(" • Full IDE support (autocomplete, types)") + print(" • Automatic conversion to API format") + print(" • Reusable across projects") + print(" • Clean, readable code") + print(" • Easy to test and debug") + print("=" * 70) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/md_v2/core/docker-deployment.md b/docs/md_v2/core/docker-deployment.md index ea3692b2..36bf28e1 100644 --- a/docs/md_v2/core/docker-deployment.md +++ b/docs/md_v2/core/docker-deployment.md @@ -6,18 +6,6 @@ - [Option 1: Using Pre-built Docker Hub Images (Recommended)](#option-1-using-pre-built-docker-hub-images-recommended) - [Option 2: Using Docker Compose](#option-2-using-docker-compose) - [Option 3: Manual Local Build & Run](#option-3-manual-local-build--run) -- [Dockerfile Parameters](#dockerfile-parameters) -- [Using the API](#using-the-api) - - [Playground Interface](#playground-interface) - - [Python SDK](#python-sdk) - - [Understanding Request Schema](#understanding-request-schema) - - [REST API Examples](#rest-api-examples) -- [Additional API Endpoints](#additional-api-endpoints) - - [HTML Extraction Endpoint](#html-extraction-endpoint) - - [Screenshot Endpoint](#screenshot-endpoint) - - [PDF Export Endpoint](#pdf-export-endpoint) - - [JavaScript Execution Endpoint](#javascript-execution-endpoint) - - [Library Context Endpoint](#library-context-endpoint) - [MCP (Model Context Protocol) Support](#mcp-model-context-protocol-support) - [What is MCP?](#what-is-mcp) - [Connecting via MCP](#connecting-via-mcp) @@ -25,9 +13,28 @@ - [Available MCP Tools](#available-mcp-tools) - [Testing MCP Connections](#testing-mcp-connections) - [MCP Schemas](#mcp-schemas) +- [Additional API Endpoints](#additional-api-endpoints) + - [HTML Extraction Endpoint](#html-extraction-endpoint) + - [Screenshot Endpoint](#screenshot-endpoint) + - [PDF Export Endpoint](#pdf-export-endpoint) + - [JavaScript Execution Endpoint](#javascript-execution-endpoint) +- [User-Provided Hooks API](#user-provided-hooks-api) + - [Hook Information Endpoint](#hook-information-endpoint) + - [Available Hook Points](#available-hook-points) + - [Using Hooks in Requests](#using-hooks-in-requests) + - [Hook Examples with Real URLs](#hook-examples-with-real-urls) + - [Security Best Practices](#security-best-practices) + - [Hook Response Information](#hook-response-information) + - [Error Handling](#error-handling) + - [Hooks Utility: Function-Based Approach (Python)](#hooks-utility-function-based-approach-python) +- [Dockerfile Parameters](#dockerfile-parameters) +- [Using the API](#using-the-api) + - [Playground Interface](#playground-interface) + - [Python SDK](#python-sdk) + - [Understanding Request Schema](#understanding-request-schema) + - [REST API Examples](#rest-api-examples) + - [LLM Configuration Examples](#llm-configuration-examples) - [Metrics & Monitoring](#metrics--monitoring) -- [Deployment Scenarios](#deployment-scenarios) -- [Complete Examples](#complete-examples) - [Server Configuration](#server-configuration) - [Understanding config.yml](#understanding-configyml) - [JWT Authentication](#jwt-authentication) @@ -832,6 +839,275 @@ else: > 💡 **Remember**: Always test your hooks on safe, known websites first before using them on production sites. Never crawl sites that you don't have permission to access or that might be malicious. +### Hooks Utility: Function-Based Approach (Python) + +For Python developers, Crawl4AI provides a more convenient way to work with hooks using the `hooks_to_string()` utility function and Docker client integration. + +#### Why Use Function-Based Hooks? + +**String-Based Approach (shown above)**: +```python +hooks_code = { + "on_page_context_created": """ +async def hook(page, context, **kwargs): + await page.set_viewport_size({"width": 1920, "height": 1080}) + return page +""" +} +``` + +**Function-Based Approach (recommended for Python)**: +```python +from crawl4ai import Crawl4aiDockerClient + +async def my_hook(page, context, **kwargs): + await page.set_viewport_size({"width": 1920, "height": 1080}) + return page + +async with Crawl4aiDockerClient(base_url="http://localhost:11235") as client: + result = await client.crawl( + ["https://example.com"], + hooks={"on_page_context_created": my_hook} + ) +``` + +**Benefits**: +- ✅ Write hooks as regular Python functions +- ✅ Full IDE support (autocomplete, syntax highlighting, type checking) +- ✅ Easy to test and debug +- ✅ Reusable hook libraries +- ✅ Automatic conversion to API format + +#### Using the Hooks Utility + +The `hooks_to_string()` utility converts Python function objects to the string format required by the API: + +```python +from crawl4ai import hooks_to_string + +# Define your hooks as functions +async def setup_hook(page, context, **kwargs): + await page.set_viewport_size({"width": 1920, "height": 1080}) + await context.add_cookies([{ + "name": "session", + "value": "token", + "domain": ".example.com" + }]) + return page + +async def scroll_hook(page, context, **kwargs): + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + return page + +# Convert to string format +hooks_dict = { + "on_page_context_created": setup_hook, + "before_retrieve_html": scroll_hook +} +hooks_string = hooks_to_string(hooks_dict) + +# Now use with REST API or Docker client +# hooks_string contains the string representations +``` + +#### Docker Client with Automatic Conversion + +The Docker client automatically detects and converts function objects: + +```python +from crawl4ai import Crawl4aiDockerClient + +async def auth_hook(page, context, **kwargs): + """Add authentication cookies""" + await context.add_cookies([{ + "name": "auth_token", + "value": "your_token", + "domain": ".example.com" + }]) + return page + +async def performance_hook(page, context, **kwargs): + """Block unnecessary resources""" + await context.route("**/*.{png,jpg,gif}", lambda r: r.abort()) + await context.route("**/analytics/*", lambda r: r.abort()) + return page + +async with Crawl4aiDockerClient(base_url="http://localhost:11235") as client: + # Pass functions directly - automatic conversion! + result = await client.crawl( + ["https://example.com"], + hooks={ + "on_page_context_created": performance_hook, + "before_goto": auth_hook + }, + hooks_timeout=30 # Optional timeout in seconds (1-120) + ) + + print(f"Success: {result.success}") + print(f"HTML: {len(result.html)} chars") +``` + +#### Creating Reusable Hook Libraries + +Build collections of reusable hooks: + +```python +# hooks_library.py +class CrawlHooks: + """Reusable hook collection for common crawling tasks""" + + @staticmethod + async def block_images(page, context, **kwargs): + """Block all images to speed up crawling""" + await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda r: r.abort()) + return page + + @staticmethod + async def block_analytics(page, context, **kwargs): + """Block analytics and tracking scripts""" + tracking_domains = [ + "**/google-analytics.com/*", + "**/googletagmanager.com/*", + "**/facebook.com/tr/*", + "**/doubleclick.net/*" + ] + for domain in tracking_domains: + await context.route(domain, lambda r: r.abort()) + return page + + @staticmethod + async def scroll_infinite(page, context, **kwargs): + """Handle infinite scroll to load more content""" + previous_height = 0 + for i in range(5): # Max 5 scrolls + current_height = await page.evaluate("document.body.scrollHeight") + if current_height == previous_height: + break + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await page.wait_for_timeout(1000) + previous_height = current_height + return page + + @staticmethod + async def wait_for_dynamic_content(page, context, url, response, **kwargs): + """Wait for dynamic content to load""" + await page.wait_for_timeout(2000) + try: + # Click "Load More" if present + load_more = await page.query_selector('[class*="load-more"]') + if load_more: + await load_more.click() + await page.wait_for_timeout(1000) + except: + pass + return page + +# Use in your application +from hooks_library import CrawlHooks +from crawl4ai import Crawl4aiDockerClient + +async def crawl_with_optimizations(url): + async with Crawl4aiDockerClient() as client: + result = await client.crawl( + [url], + hooks={ + "on_page_context_created": CrawlHooks.block_images, + "before_retrieve_html": CrawlHooks.scroll_infinite + } + ) + return result +``` + +#### Choosing the Right Approach + +| Approach | Best For | IDE Support | Language | +|----------|----------|-------------|----------| +| **String-based** | Non-Python clients, REST APIs, other languages | ❌ None | Any | +| **Function-based** | Python applications, local development | ✅ Full | Python only | +| **Docker Client** | Python apps with automatic conversion | ✅ Full | Python only | + +**Recommendation**: +- **Python applications**: Use Docker client with function objects (easiest) +- **Non-Python or REST API**: Use string-based hooks (most flexible) +- **Manual control**: Use `hooks_to_string()` utility (middle ground) + +#### Complete Example with Function Hooks + +```python +from crawl4ai import Crawl4aiDockerClient, BrowserConfig, CrawlerRunConfig, CacheMode + +# Define hooks as regular Python functions +async def setup_environment(page, context, **kwargs): + """Setup crawling environment""" + # Set viewport + await page.set_viewport_size({"width": 1920, "height": 1080}) + + # Block resources for speed + await context.route("**/*.{png,jpg,gif}", lambda r: r.abort()) + + # Add custom headers + await page.set_extra_http_headers({ + "Accept-Language": "en-US", + "X-Custom-Header": "Crawl4AI" + }) + + print("[HOOK] Environment configured") + return page + +async def extract_content(page, context, **kwargs): + """Extract and prepare content""" + # Scroll to load lazy content + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await page.wait_for_timeout(1000) + + # Extract metadata + metadata = await page.evaluate('''() => ({ + title: document.title, + links: document.links.length, + images: document.images.length + })''') + + print(f"[HOOK] Page metadata: {metadata}") + return page + +async def main(): + async with Crawl4aiDockerClient(base_url="http://localhost:11235", verbose=True) as client: + # Configure crawl + browser_config = BrowserConfig(headless=True) + crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS) + + # Crawl with hooks + result = await client.crawl( + ["https://httpbin.org/html"], + browser_config=browser_config, + crawler_config=crawler_config, + hooks={ + "on_page_context_created": setup_environment, + "before_retrieve_html": extract_content + }, + hooks_timeout=30 + ) + + if result.success: + print(f"✅ Crawl successful!") + print(f" URL: {result.url}") + print(f" HTML: {len(result.html)} chars") + print(f" Markdown: {len(result.markdown)} chars") + else: + print(f"❌ Crawl failed: {result.error_message}") + +if __name__ == "__main__": + import asyncio + asyncio.run(main()) +``` + +#### Additional Resources + +- **Comprehensive Examples**: See `/docs/examples/hooks_docker_client_example.py` for Python function-based examples +- **REST API Examples**: See `/docs/examples/hooks_rest_api_example.py` for string-based examples +- **Comparison Guide**: See `/docs/examples/README_HOOKS.md` for detailed comparison +- **Utility Documentation**: See `/docs/hooks-utility-guide.md` for complete guide + --- ## Dockerfile Parameters @@ -892,10 +1168,12 @@ This is the easiest way to translate Python configuration to JSON requests when Install the SDK: `pip install crawl4ai` +The Python SDK provides a convenient way to interact with the Docker API, including **automatic hook conversion** when using function objects. + ```python import asyncio from crawl4ai.docker_client import Crawl4aiDockerClient -from crawl4ai import BrowserConfig, CrawlerRunConfig, CacheMode # Assuming you have crawl4ai installed +from crawl4ai import BrowserConfig, CrawlerRunConfig, CacheMode async def main(): # Point to the correct server port @@ -907,23 +1185,22 @@ async def main(): print("--- Running Non-Streaming Crawl ---") results = await client.crawl( ["https://httpbin.org/html"], - browser_config=BrowserConfig(headless=True), # Use library classes for config aid + browser_config=BrowserConfig(headless=True), crawler_config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS) ) - if results: # client.crawl returns None on failure - print(f"Non-streaming results success: {results.success}") - if results.success: - for result in results: # Iterate through the CrawlResultContainer - print(f"URL: {result.url}, Success: {result.success}") + if results: + print(f"Non-streaming results success: {results.success}") + if results.success: + for result in results: + print(f"URL: {result.url}, Success: {result.success}") else: print("Non-streaming crawl failed.") - # Example Streaming crawl print("\n--- Running Streaming Crawl ---") stream_config = CrawlerRunConfig(stream=True, cache_mode=CacheMode.BYPASS) try: - async for result in await client.crawl( # client.crawl returns an async generator for streaming + async for result in await client.crawl( ["https://httpbin.org/html", "https://httpbin.org/links/5/0"], browser_config=BrowserConfig(headless=True), crawler_config=stream_config @@ -932,17 +1209,56 @@ async def main(): except Exception as e: print(f"Streaming crawl failed: {e}") + # Example with hooks (Python function objects) + print("\n--- Crawl with Hooks ---") + + async def my_hook(page, context, **kwargs): + """Custom hook to optimize performance""" + await page.set_viewport_size({"width": 1920, "height": 1080}) + await context.route("**/*.{png,jpg}", lambda r: r.abort()) + print("[HOOK] Page optimized") + return page + + result = await client.crawl( + ["https://httpbin.org/html"], + browser_config=BrowserConfig(headless=True), + crawler_config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS), + hooks={"on_page_context_created": my_hook}, # Pass function directly! + hooks_timeout=30 + ) + print(f"Crawl with hooks success: {result.success}") # Example Get schema print("\n--- Getting Schema ---") schema = await client.get_schema() - print(f"Schema received: {bool(schema)}") # Print whether schema was received + print(f"Schema received: {bool(schema)}") if __name__ == "__main__": asyncio.run(main()) ``` -*(SDK parameters like timeout, verify_ssl etc. remain the same)* +#### SDK Parameters + +The Docker client supports the following parameters: + +**Client Initialization**: +- `base_url` (str): URL of the Docker server (default: `http://localhost:8000`) +- `timeout` (float): Request timeout in seconds (default: 30.0) +- `verify_ssl` (bool): Verify SSL certificates (default: True) +- `verbose` (bool): Enable verbose logging (default: True) +- `log_file` (Optional[str]): Path to log file (default: None) + +**crawl() Method**: +- `urls` (List[str]): List of URLs to crawl +- `browser_config` (Optional[BrowserConfig]): Browser configuration +- `crawler_config` (Optional[CrawlerRunConfig]): Crawler configuration +- `hooks` (Optional[Dict]): Hook functions or strings - **automatically converts function objects!** +- `hooks_timeout` (int): Timeout for each hook execution in seconds (default: 30) + +**Returns**: +- Single URL: `CrawlResult` object +- Multiple URLs: `List[CrawlResult]` +- Streaming: `AsyncGenerator[CrawlResult]` ### Second Approach: Direct API Calls @@ -1352,19 +1668,40 @@ We're here to help you succeed with Crawl4AI! Here's how to get support: In this guide, we've covered everything you need to get started with Crawl4AI's Docker deployment: - Building and running the Docker container -- Configuring the environment +- Configuring the environment - Using the interactive playground for testing - Making API requests with proper typing -- Using the Python SDK +- Using the Python SDK with **automatic hook conversion** +- **Working with hooks** - both string-based (REST API) and function-based (Python SDK) - Leveraging specialized endpoints for screenshots, PDFs, and JavaScript execution - Connecting via the Model Context Protocol (MCP) - Monitoring your deployment -The new playground interface at `http://localhost:11235/playground` makes it much easier to test configurations and generate the corresponding JSON for API requests. +### Key Features -For AI application developers, the MCP integration allows tools like Claude Code to directly access Crawl4AI's capabilities without complex API handling. +**Hooks Support**: Crawl4AI offers two approaches for working with hooks: +- **String-based** (REST API): Works with any language, requires manual string formatting +- **Function-based** (Python SDK): Write hooks as regular Python functions with full IDE support and automatic conversion -Remember, the examples in the `examples` folder are your friends - they show real-world usage patterns that you can adapt for your needs. +**Playground Interface**: The built-in playground at `http://localhost:11235/playground` makes it easy to test configurations and generate corresponding JSON for API requests. + +**MCP Integration**: For AI application developers, the MCP integration allows tools like Claude Code to directly access Crawl4AI's capabilities without complex API handling. + +### Next Steps + +1. **Explore Examples**: Check out the comprehensive examples in: + - `/docs/examples/hooks_docker_client_example.py` - Python function-based hooks + - `/docs/examples/hooks_rest_api_example.py` - REST API string-based hooks + - `/docs/examples/README_HOOKS.md` - Comparison and guide + +2. **Read Documentation**: + - `/docs/hooks-utility-guide.md` - Complete hooks utility guide + - API documentation for detailed configuration options + +3. **Join the Community**: + - GitHub: Report issues and contribute + - Discord: Get help and share your experiences + - Documentation: Comprehensive guides and tutorials Keep exploring, and don't hesitate to reach out if you need help! We're building something amazing together. 🚀 diff --git a/tests/docker/test_hooks_utility.py b/tests/docker/test_hooks_utility.py new file mode 100644 index 00000000..7c820e56 --- /dev/null +++ b/tests/docker/test_hooks_utility.py @@ -0,0 +1,193 @@ +""" +Test script demonstrating the hooks_to_string utility and Docker client integration. +""" +import asyncio +from crawl4ai import Crawl4aiDockerClient, hooks_to_string + + +# Define hook functions as regular Python functions +async def auth_hook(page, context, **kwargs): + """Add authentication cookies.""" + await context.add_cookies([{ + 'name': 'test_cookie', + 'value': 'test_value', + 'domain': '.httpbin.org', + 'path': '/' + }]) + return page + + +async def scroll_hook(page, context, **kwargs): + """Scroll to load lazy content.""" + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await page.wait_for_timeout(1000) + return page + + +async def viewport_hook(page, context, **kwargs): + """Set custom viewport.""" + await page.set_viewport_size({"width": 1920, "height": 1080}) + return page + + +async def test_hooks_utility(): + """Test the hooks_to_string utility function.""" + print("=" * 60) + print("Testing hooks_to_string utility") + print("=" * 60) + + # Create hooks dictionary with function objects + hooks_dict = { + "on_page_context_created": auth_hook, + "before_retrieve_html": scroll_hook + } + + # Convert to string format + hooks_string = hooks_to_string(hooks_dict) + + print("\n✓ Successfully converted function objects to strings") + print(f"\n✓ Converted {len(hooks_string)} hooks:") + for hook_name in hooks_string.keys(): + print(f" - {hook_name}") + + print("\n✓ Preview of converted hook:") + print("-" * 60) + print(hooks_string["on_page_context_created"][:200] + "...") + print("-" * 60) + + return hooks_string + + +async def test_docker_client_with_functions(): + """Test Docker client with function objects (automatic conversion).""" + print("\n" + "=" * 60) + print("Testing Docker Client with Function Objects") + print("=" * 60) + + # Note: This requires a running Crawl4AI Docker server + # Uncomment the following to test with actual server: + + async with Crawl4aiDockerClient(base_url="http://localhost:11234", verbose=True) as client: + # Pass function objects directly - they'll be converted automatically + result = await client.crawl( + ["https://httpbin.org/html"], + hooks={ + "on_page_context_created": auth_hook, + "before_retrieve_html": scroll_hook + }, + hooks_timeout=30 + ) + print(f"\n✓ Crawl successful: {result.success}") + print(f"✓ URL: {result.url}") + + print("\n✓ Docker client accepts function objects directly") + print("✓ Automatic conversion happens internally") + print("✓ No manual string formatting needed!") + + +async def test_docker_client_with_strings(): + """Test Docker client with pre-converted strings.""" + print("\n" + "=" * 60) + print("Testing Docker Client with String Hooks") + print("=" * 60) + + # Convert hooks to strings first + hooks_dict = { + "on_page_context_created": viewport_hook, + "before_retrieve_html": scroll_hook + } + hooks_string = hooks_to_string(hooks_dict) + + # Note: This requires a running Crawl4AI Docker server + # Uncomment the following to test with actual server: + + async with Crawl4aiDockerClient(base_url="http://localhost:11234", verbose=True) as client: + # Pass string hooks - they'll be used as-is + result = await client.crawl( + ["https://httpbin.org/html"], + hooks=hooks_string, + hooks_timeout=30 + ) + print(f"\n✓ Crawl successful: {result.success}") + + print("\n✓ Docker client also accepts pre-converted strings") + print("✓ Backward compatible with existing code") + + +async def show_usage_patterns(): + """Show different usage patterns.""" + print("\n" + "=" * 60) + print("Usage Patterns") + print("=" * 60) + + print("\n1. Direct function usage (simplest):") + print("-" * 60) + print(""" + async def my_hook(page, context, **kwargs): + await page.set_viewport_size({"width": 1920, "height": 1080}) + return page + + result = await client.crawl( + ["https://example.com"], + hooks={"on_page_context_created": my_hook} + ) + """) + + print("\n2. Convert then use:") + print("-" * 60) + print(""" + hooks_dict = {"on_page_context_created": my_hook} + hooks_string = hooks_to_string(hooks_dict) + + result = await client.crawl( + ["https://example.com"], + hooks=hooks_string + ) + """) + + print("\n3. Manual string (backward compatible):") + print("-" * 60) + print(""" + hooks_string = { + "on_page_context_created": ''' +async def hook(page, context, **kwargs): + await page.set_viewport_size({"width": 1920, "height": 1080}) + return page +''' + } + + result = await client.crawl( + ["https://example.com"], + hooks=hooks_string + ) + """) + + +async def main(): + """Run all tests.""" + print("\n🚀 Crawl4AI Hooks Utility Test Suite\n") + + # Test the utility function + # await test_hooks_utility() + + # Show usage with Docker client + # await test_docker_client_with_functions() + await test_docker_client_with_strings() + + # Show different patterns + # await show_usage_patterns() + + # print("\n" + "=" * 60) + # print("✓ All tests completed successfully!") + # print("=" * 60) + # print("\nKey Benefits:") + # print(" • Write hooks as regular Python functions") + # print(" • IDE support with autocomplete and type checking") + # print(" • Automatic conversion to API format") + # print(" • Backward compatible with string hooks") + # print(" • Same utility used everywhere") + # print("\n") + + +if __name__ == "__main__": + asyncio.run(main()) From 4a04b8506a81d28afa2f8039554a00f1e6bf588b Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 13 Oct 2025 12:53:33 +0800 Subject: [PATCH 054/119] feat: Add hooks utility for function-based hooks with Docker client integration. ref #1377 Add hooks_to_string() utility function that converts Python function objects to string representations for the Docker API, enabling developers to write hooks as regular Python functions instead of strings. Core Changes: - New hooks_to_string() utility in crawl4ai/utils.py using inspect.getsource() - Docker client now accepts both function objects and strings for hooks - Automatic detection and conversion in Crawl4aiDockerClient._prepare_request() - New hooks and hooks_timeout parameters in client.crawl() method Documentation: - Docker client examples with function-based hooks (docs/examples/docker_client_hooks_example.py) - Updated main Docker deployment guide with comprehensive hooks section - Added unit tests for hooks utility (tests/docker/test_hooks_utility.py) --- docs/blog/release-v0.7.5.md | 82 +++++++++++- docs/md_v2/blog/releases/v0.7.5.md | 82 +++++++++++- docs/releases_review/demo_v0.7.5.py | 185 ++++++++++++++++------------ 3 files changed, 269 insertions(+), 80 deletions(-) diff --git a/docs/blog/release-v0.7.5.md b/docs/blog/release-v0.7.5.md index 5740873f..977d2fd9 100644 --- a/docs/blog/release-v0.7.5.md +++ b/docs/blog/release-v0.7.5.md @@ -8,7 +8,8 @@ Today I'm releasing Crawl4AI v0.7.5—focused on extensibility and security. Thi ## 🎯 What's New at a Glance -- **Docker Hooks System**: Custom Python functions at key pipeline points +- **Docker Hooks System**: Custom Python functions at key pipeline points with function-based API +- **Function-Based Hooks**: New `hooks_to_string()` utility with Docker client auto-conversion - **Enhanced LLM Integration**: Custom providers with temperature control - **HTTPS Preservation**: Secure internal link handling - **Bug Fixes**: Resolved multiple community-reported issues @@ -82,6 +83,85 @@ if result.get('success'): - `before_retrieve_html`: Pre-extraction processing - `before_return_html`: Final HTML processing +### Function-Based Hooks API + +Writing hooks as strings works, but lacks IDE support and type checking. v0.7.5 introduces a function-based approach with automatic conversion! + +**Option 1: Using the `hooks_to_string()` Utility** + +```python +from crawl4ai import hooks_to_string +import requests + +# Define hooks as regular Python functions (with full IDE support!) +async def on_page_context_created(page, context, **kwargs): + """Block images to speed up crawling""" + await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort()) + await page.set_viewport_size({"width": 1920, "height": 1080}) + return page + +async def before_goto(page, context, url, **kwargs): + """Add custom headers""" + await page.set_extra_http_headers({ + 'X-Crawl4AI': 'v0.7.5', + 'X-Custom-Header': 'my-value' + }) + return page + +# Convert functions to strings +hooks_code = hooks_to_string({ + "on_page_context_created": on_page_context_created, + "before_goto": before_goto +}) + +# Use with REST API +payload = { + "urls": ["https://httpbin.org/html"], + "hooks": {"code": hooks_code, "timeout": 30} +} +response = requests.post("http://localhost:11235/crawl", json=payload) +``` + +**Option 2: Docker Client with Automatic Conversion (Recommended!)** + +```python +from crawl4ai.docker_client import Crawl4aiDockerClient + +# Define hooks as functions (same as above) +async def on_page_context_created(page, context, **kwargs): + await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort()) + return page + +async def before_retrieve_html(page, context, **kwargs): + # Scroll to load lazy content + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await page.wait_for_timeout(1000) + return page + +# Use Docker client - conversion happens automatically! +client = Crawl4aiDockerClient(base_url="http://localhost:11235") + +results = await client.crawl( + urls=["https://httpbin.org/html"], + hooks={ + "on_page_context_created": on_page_context_created, + "before_retrieve_html": before_retrieve_html + }, + hooks_timeout=30 +) + +if results and results.success: + print(f"✅ Hooks executed! HTML length: {len(results.html)}") +``` + +**Benefits of Function-Based Hooks:** +- ✅ Full IDE support (autocomplete, syntax highlighting) +- ✅ Type checking and linting +- ✅ Easier to test and debug +- ✅ Reusable across projects +- ✅ Automatic conversion in Docker client +- ✅ No breaking changes - string hooks still work! + ## 🤖 Enhanced LLM Integration Enhanced LLM integration with custom providers, temperature control, and base URL configuration. diff --git a/docs/md_v2/blog/releases/v0.7.5.md b/docs/md_v2/blog/releases/v0.7.5.md index 5740873f..977d2fd9 100644 --- a/docs/md_v2/blog/releases/v0.7.5.md +++ b/docs/md_v2/blog/releases/v0.7.5.md @@ -8,7 +8,8 @@ Today I'm releasing Crawl4AI v0.7.5—focused on extensibility and security. Thi ## 🎯 What's New at a Glance -- **Docker Hooks System**: Custom Python functions at key pipeline points +- **Docker Hooks System**: Custom Python functions at key pipeline points with function-based API +- **Function-Based Hooks**: New `hooks_to_string()` utility with Docker client auto-conversion - **Enhanced LLM Integration**: Custom providers with temperature control - **HTTPS Preservation**: Secure internal link handling - **Bug Fixes**: Resolved multiple community-reported issues @@ -82,6 +83,85 @@ if result.get('success'): - `before_retrieve_html`: Pre-extraction processing - `before_return_html`: Final HTML processing +### Function-Based Hooks API + +Writing hooks as strings works, but lacks IDE support and type checking. v0.7.5 introduces a function-based approach with automatic conversion! + +**Option 1: Using the `hooks_to_string()` Utility** + +```python +from crawl4ai import hooks_to_string +import requests + +# Define hooks as regular Python functions (with full IDE support!) +async def on_page_context_created(page, context, **kwargs): + """Block images to speed up crawling""" + await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort()) + await page.set_viewport_size({"width": 1920, "height": 1080}) + return page + +async def before_goto(page, context, url, **kwargs): + """Add custom headers""" + await page.set_extra_http_headers({ + 'X-Crawl4AI': 'v0.7.5', + 'X-Custom-Header': 'my-value' + }) + return page + +# Convert functions to strings +hooks_code = hooks_to_string({ + "on_page_context_created": on_page_context_created, + "before_goto": before_goto +}) + +# Use with REST API +payload = { + "urls": ["https://httpbin.org/html"], + "hooks": {"code": hooks_code, "timeout": 30} +} +response = requests.post("http://localhost:11235/crawl", json=payload) +``` + +**Option 2: Docker Client with Automatic Conversion (Recommended!)** + +```python +from crawl4ai.docker_client import Crawl4aiDockerClient + +# Define hooks as functions (same as above) +async def on_page_context_created(page, context, **kwargs): + await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort()) + return page + +async def before_retrieve_html(page, context, **kwargs): + # Scroll to load lazy content + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await page.wait_for_timeout(1000) + return page + +# Use Docker client - conversion happens automatically! +client = Crawl4aiDockerClient(base_url="http://localhost:11235") + +results = await client.crawl( + urls=["https://httpbin.org/html"], + hooks={ + "on_page_context_created": on_page_context_created, + "before_retrieve_html": before_retrieve_html + }, + hooks_timeout=30 +) + +if results and results.success: + print(f"✅ Hooks executed! HTML length: {len(results.html)}") +``` + +**Benefits of Function-Based Hooks:** +- ✅ Full IDE support (autocomplete, syntax highlighting) +- ✅ Type checking and linting +- ✅ Easier to test and debug +- ✅ Reusable across projects +- ✅ Automatic conversion in Docker client +- ✅ No breaking changes - string hooks still work! + ## 🤖 Enhanced LLM Integration Enhanced LLM integration with custom providers, temperature control, and base URL configuration. diff --git a/docs/releases_review/demo_v0.7.5.py b/docs/releases_review/demo_v0.7.5.py index d25778ee..bda472ab 100644 --- a/docs/releases_review/demo_v0.7.5.py +++ b/docs/releases_review/demo_v0.7.5.py @@ -4,7 +4,7 @@ This demo showcases key features introduced in v0.7.5 with real, executable examples. Featured Demos: -1. ✅ Docker Hooks System - Real API calls with custom hooks +1. ✅ Docker Hooks System - Real API calls with custom hooks (string & function-based) 2. ✅ Enhanced LLM Integration - Working LLM configurations 3. ✅ HTTPS Preservation - Live crawling with HTTPS maintenance @@ -19,8 +19,10 @@ import requests import time import sys -from crawl4ai import (AsyncWebCrawler, CrawlerRunConfig, BrowserConfig, - CacheMode, FilterChain, URLPatternFilter, BFSDeepCrawlStrategy) +from crawl4ai import (AsyncWebCrawler, CrawlerRunConfig, BrowserConfig, + CacheMode, FilterChain, URLPatternFilter, BFSDeepCrawlStrategy, + hooks_to_string) +from crawl4ai.docker_client import Crawl4aiDockerClient def print_section(title: str, description: str = ""): @@ -36,13 +38,13 @@ async def demo_1_docker_hooks_system(): """Demo 1: Docker Hooks System - Real API calls with custom hooks""" print_section( "Demo 1: Docker Hooks System", - "Testing real Docker hooks with live API calls" + "Testing both string-based and function-based hooks (NEW in v0.7.5!)" ) # Check Docker service availability def check_docker_service(): try: - response = requests.get("http://localhost:11234/", timeout=3) + response = requests.get("http://localhost:11235/", timeout=3) return response.status_code == 200 except: return False @@ -60,108 +62,132 @@ async def demo_1_docker_hooks_system(): print("✓ Docker service detected!") - # Define real working hooks - hooks_config = { + # ============================================================================ + # PART 1: Traditional String-Based Hooks (Works with REST API) + # ============================================================================ + print("\n" + "─" * 60) + print("Part 1: String-Based Hooks (REST API)") + print("─" * 60) + + hooks_config_string = { "on_page_context_created": """ async def hook(page, context, **kwargs): - print("Hook: Setting up page context") - # Block images to speed up crawling + print("[String Hook] Setting up page context") await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort()) - print("Hook: Images blocked") return page """, - "before_retrieve_html": """ async def hook(page, context, **kwargs): - print("Hook: Before retrieving HTML") - # Scroll to bottom to load lazy content + print("[String Hook] Before retrieving HTML") await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") await page.wait_for_timeout(1000) - print("Hook: Scrolled to bottom") - return page -""", - - "before_goto": """ -async def hook(page, context, url, **kwargs): - print(f"Hook: About to navigate to {url}") - # Add custom headers - await page.set_extra_http_headers({ - 'X-Test-Header': 'crawl4ai-hooks-test' - }) return page """ } - # Test with a reliable URL - test_url = "https://httpbin.org/html" - payload = { "urls": ["https://httpbin.org/html"], "hooks": { - "code": hooks_config, + "code": hooks_config_string, "timeout": 30 } } - print(f"🎯 Testing URL: {test_url}") - print("🔧 Configured 3 hooks: on_page_context_created, before_retrieve_html, before_goto\n") - - # Make the request - print("🔄 Executing hooks...") - + print("🔧 Using string-based hooks for REST API...") try: start_time = time.time() - response = requests.post( - "http://localhost:11234/crawl", - json=payload, - timeout=60 - ) + response = requests.post("http://localhost:11235/crawl", json=payload, timeout=60) execution_time = time.time() - start_time if response.status_code == 200: result = response.json() - - print(f"🎉 Success! Execution time: {execution_time:.2f}s\n") - - # Display results - success = result.get('success', False) - print(f"✅ Crawl Status: {'Success' if success else 'Failed'}") - - if success: - markdown_content = result.get('markdown', '') - print(f"📄 Content Length: {len(markdown_content)} characters") - - # Show content preview - if markdown_content: - preview = markdown_content[:300] + "..." if len(markdown_content) > 300 else markdown_content - print("\n--- Content Preview ---") - print(preview) - print("--- End Preview ---\n") - - # Check if our hook marker is present - raw_html = result.get('html', '') - if "Crawl4AI v0.7.5 Docker Hook" in raw_html: - print("✓ Hook marker found in HTML - hooks executed successfully!") - - # Display hook execution info if available - print("\nHook Execution Summary:") - print("🔗 before_goto: URL modified with tracking parameter") - print("✅ after_goto: Page navigation completed") - print("📝 before_return_html: Content processed and marked") - + print(f"✅ String-based hooks executed in {execution_time:.2f}s") + if result.get('results') and result['results'][0].get('success'): + html_length = len(result['results'][0].get('html', '')) + print(f" 📄 HTML length: {html_length} characters") else: print(f"❌ Request failed: {response.status_code}") - try: - error_data = response.json() - print(f"Error: {error_data}") - except: - print(f"Raw response: {response.text[:500]}") - - except requests.exceptions.Timeout: - print("⏰ Request timed out after 60 seconds") except Exception as e: print(f"❌ Error: {str(e)}") + # ============================================================================ + # PART 2: NEW Function-Based Hooks with Docker Client (v0.7.5) + # ============================================================================ + print("\n" + "─" * 60) + print("Part 2: Function-Based Hooks with Docker Client (✨ NEW!)") + print("─" * 60) + + # Define hooks as regular Python functions + async def on_page_context_created_func(page, context, **kwargs): + """Block images to speed up crawling""" + print("[Function Hook] Setting up page context") + await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort()) + await page.set_viewport_size({"width": 1920, "height": 1080}) + return page + + async def before_goto_func(page, context, url, **kwargs): + """Add custom headers before navigation""" + print(f"[Function Hook] About to navigate to {url}") + await page.set_extra_http_headers({ + 'X-Crawl4AI': 'v0.7.5-function-hooks', + 'X-Test-Header': 'demo' + }) + return page + + async def before_retrieve_html_func(page, context, **kwargs): + """Scroll to load lazy content""" + print("[Function Hook] Scrolling page for lazy-loaded content") + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await page.wait_for_timeout(500) + await page.evaluate("window.scrollTo(0, 0)") + return page + + # Use the hooks_to_string utility (can be used standalone) + print("\n📦 Converting functions to strings with hooks_to_string()...") + hooks_as_strings = hooks_to_string({ + "on_page_context_created": on_page_context_created_func, + "before_goto": before_goto_func, + "before_retrieve_html": before_retrieve_html_func + }) + print(f" ✓ Converted {len(hooks_as_strings)} hooks to string format") + + # OR use Docker Client which does conversion automatically! + print("\n🐳 Using Docker Client with automatic conversion...") + try: + client = Crawl4aiDockerClient(base_url="http://localhost:11235") + + # Pass function objects directly - conversion happens automatically! + results = await client.crawl( + urls=["https://httpbin.org/html"], + hooks={ + "on_page_context_created": on_page_context_created_func, + "before_goto": before_goto_func, + "before_retrieve_html": before_retrieve_html_func + }, + hooks_timeout=30 + ) + + if results and results.success: + print(f"✅ Function-based hooks executed successfully!") + print(f" 📄 HTML length: {len(results.html)} characters") + print(f" 🎯 URL: {results.url}") + else: + print("⚠️ Crawl completed but may have warnings") + + except Exception as e: + print(f"❌ Docker client error: {str(e)}") + + # Show the benefits + print("\n" + "=" * 60) + print("✨ Benefits of Function-Based Hooks:") + print("=" * 60) + print("✓ Full IDE support (autocomplete, syntax highlighting)") + print("✓ Type checking and linting") + print("✓ Easier to test and debug") + print("✓ Reusable across projects") + print("✓ Automatic conversion in Docker client") + print("=" * 60) + async def demo_2_enhanced_llm_integration(): """Demo 2: Enhanced LLM Integration - Working LLM configurations""" @@ -182,7 +208,7 @@ async def demo_2_enhanced_llm_integration(): } try: response = requests.post( - "http://localhost:11234/md", + "http://localhost:11235/md", json=payload, timeout=60 ) @@ -285,7 +311,10 @@ async def main(): print("You've experienced the power of Crawl4AI v0.7.5!") print("") print("Key Features Demonstrated:") - print("🔧 Docker Hooks - Custom pipeline modifications") + print("🔧 Docker Hooks - String-based & function-based (NEW!)") + print(" • hooks_to_string() utility for function conversion") + print(" • Docker client with automatic conversion") + print(" • Full IDE support and type checking") print("🤖 Enhanced LLM - Better AI integration") print("🔒 HTTPS Preservation - Secure link handling") print("") From aadab30c3dc8f5d92aa754ff2dc03ce0d9260621 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 13 Oct 2025 13:08:47 +0800 Subject: [PATCH 055/119] fix(docs): clarify Docker Hooks System with function-based API in README --- README.md | 56 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 58d4bf4c..ef0002e1 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data [✨ Check out latest update v0.7.5](#-recent-updates) -✨ New in v0.7.5: Docker Hooks System for pipeline customization, Enhanced LLM Integration with custom providers, HTTPS Preservation, and multiple community-reported bug fixes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.5.md) +✨ New in v0.7.5: Docker Hooks System with function-based API for pipeline customization, Enhanced LLM Integration with custom providers, HTTPS Preservation, and multiple community-reported bug fixes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.5.md) ✨ Recent v0.7.4: Revolutionary LLM Table Extraction with intelligent chunking, enhanced concurrency fixes, memory management refactor, and critical stability improvements. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.4.md) @@ -179,7 +179,7 @@ No rate-limited APIs. No lock-in. Build and own your data pipeline with direct g - 📸 **Screenshots**: Capture page screenshots during crawling for debugging or analysis. - 📂 **Raw Data Crawling**: Directly process raw HTML (`raw:`) or local files (`file://`). - 🔗 **Comprehensive Link Extraction**: Extracts internal, external links, and embedded iframe content. -- 🛠️ **Customizable Hooks**: Define hooks at every step to customize crawling behavior. +- 🛠️ **Customizable Hooks**: Define hooks at every step to customize crawling behavior (supports both string and function-based APIs). - 💾 **Caching**: Cache data for improved speed and to avoid redundant fetches. - 📄 **Metadata Extraction**: Retrieve structured metadata from web pages. - 📡 **IFrame Content Extraction**: Seamless extraction from embedded iframe content. @@ -549,34 +549,40 @@ async def test_news_crawl():
Version 0.7.5 Release Highlights - The Docker Hooks & Security Update -- **🔧 Docker Hooks System**: Complete pipeline customization with user-provided Python functions: +- **🔧 Docker Hooks System**: Complete pipeline customization with user-provided Python functions at 8 key points +- **✨ Function-Based Hooks API (NEW)**: Write hooks as regular Python functions with full IDE support: ```python - import requests + from crawl4ai import hooks_to_string + from crawl4ai.docker_client import Crawl4aiDockerClient - # Real working hooks for httpbin.org - hooks_config = { - "on_page_context_created": """ - async def hook(page, context, **kwargs): - print("Hook: Setting up page context") - # Block images to speed up crawling + # Define hooks as regular Python functions + async def on_page_context_created(page, context, **kwargs): + """Block images to speed up crawling""" await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort()) + await page.set_viewport_size({"width": 1920, "height": 1080}) return page - """, - "before_goto": """ - async def hook(page, context, url, **kwargs): - print(f"Hook: About to navigate to {url}") - # Add custom headers - await page.set_extra_http_headers({'X-Test-Header': 'crawl4ai-hooks-test'}) - return page - """ - } - # Test with Docker API - payload = { - "urls": ["https://httpbin.org/html"], - "hooks": {"code": hooks_config, "timeout": 30} - } - response = requests.post("http://localhost:11235/crawl", json=payload) + async def before_goto(page, context, url, **kwargs): + """Add custom headers""" + await page.set_extra_http_headers({'X-Crawl4AI': 'v0.7.5'}) + return page + + # Option 1: Use hooks_to_string() utility for REST API + hooks_code = hooks_to_string({ + "on_page_context_created": on_page_context_created, + "before_goto": before_goto + }) + + # Option 2: Docker client with automatic conversion (Recommended) + client = Crawl4aiDockerClient(base_url="http://localhost:11235") + results = await client.crawl( + urls=["https://httpbin.org/html"], + hooks={ + "on_page_context_created": on_page_context_created, + "before_goto": before_goto + } + ) + # ✓ Full IDE support, type checking, and reusability! ``` - **🤖 Enhanced LLM Integration**: Custom providers with temperature control and base_url configuration From 8fc1747225a887efe9e86130b5e9aebb058f24fa Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 13 Oct 2025 13:59:34 +0800 Subject: [PATCH 056/119] docs: Add demonstration files for v0.7.5 release, showcasing the new Docker Hooks System and all other features. --- .../v0.7.5_docker_hooks_demo.py | 655 +++++++ .../v0.7.5_video_walkthrough.ipynb | 1516 +++++++++++++++++ 2 files changed, 2171 insertions(+) create mode 100644 docs/releases_review/v0.7.5_docker_hooks_demo.py create mode 100644 docs/releases_review/v0.7.5_video_walkthrough.ipynb diff --git a/docs/releases_review/v0.7.5_docker_hooks_demo.py b/docs/releases_review/v0.7.5_docker_hooks_demo.py new file mode 100644 index 00000000..9b4be0c2 --- /dev/null +++ b/docs/releases_review/v0.7.5_docker_hooks_demo.py @@ -0,0 +1,655 @@ +#!/usr/bin/env python3 +""" +🚀 Crawl4AI v0.7.5 - Docker Hooks System Complete Demonstration +================================================================ + +This file demonstrates the NEW Docker Hooks System introduced in v0.7.5. + +The Docker Hooks System is a completely NEW feature that provides pipeline +customization through user-provided Python functions. It offers three approaches: + +1. String-based hooks for REST API +2. hooks_to_string() utility to convert functions +3. Docker Client with automatic conversion (most convenient) + +All three approaches are part of this NEW v0.7.5 feature! + +Perfect for video recording and demonstration purposes. + +Requirements: +- Docker container running: docker run -p 11235:11235 unclecode/crawl4ai:latest +- crawl4ai v0.7.5 installed: pip install crawl4ai==0.7.5 +""" + +import asyncio +import requests +import json +import time +from typing import Dict, Any + +# Import Crawl4AI components +from crawl4ai import hooks_to_string +from crawl4ai.docker_client import Crawl4aiDockerClient + +# Configuration +# DOCKER_URL = "http://localhost:11235" +DOCKER_URL = "http://localhost:11234" +TEST_URLS = [ + # "https://httpbin.org/html", + "https://www.kidocode.com", + "https://quotes.toscrape.com", +] + + +def print_section(title: str, description: str = ""): + """Print a formatted section header""" + print("\n" + "=" * 70) + print(f" {title}") + if description: + print(f" {description}") + print("=" * 70 + "\n") + + +def check_docker_service() -> bool: + """Check if Docker service is running""" + try: + response = requests.get(f"{DOCKER_URL}/health", timeout=3) + return response.status_code == 200 + except: + return False + + +# ============================================================================ +# REUSABLE HOOK LIBRARY (NEW in v0.7.5) +# ============================================================================ + +async def performance_optimization_hook(page, context, **kwargs): + """ + Performance Hook: Block unnecessary resources to speed up crawling + """ + print(" [Hook] 🚀 Optimizing performance - blocking images and ads...") + + # Block images + await context.route( + "**/*.{png,jpg,jpeg,gif,webp,svg,ico}", + lambda route: route.abort() + ) + + # Block ads and analytics + await context.route("**/analytics/*", lambda route: route.abort()) + await context.route("**/ads/*", lambda route: route.abort()) + await context.route("**/google-analytics.com/*", lambda route: route.abort()) + + print(" [Hook] ✓ Performance optimization applied") + return page + + +async def viewport_setup_hook(page, context, **kwargs): + """ + Viewport Hook: Set consistent viewport size for rendering + """ + print(" [Hook] 🖥️ Setting viewport to 1920x1080...") + await page.set_viewport_size({"width": 1920, "height": 1080}) + print(" [Hook] ✓ Viewport configured") + return page + + +async def authentication_headers_hook(page, context, url, **kwargs): + """ + Headers Hook: Add custom authentication and tracking headers + """ + print(f" [Hook] 🔐 Adding custom headers for {url[:50]}...") + + await page.set_extra_http_headers({ + 'X-Crawl4AI-Version': '0.7.5', + 'X-Custom-Hook': 'function-based-demo', + 'Accept-Language': 'en-US,en;q=0.9', + 'User-Agent': 'Crawl4AI/0.7.5 (Educational Demo)' + }) + + print(" [Hook] ✓ Custom headers added") + return page + + +async def lazy_loading_handler_hook(page, context, **kwargs): + """ + Content Hook: Handle lazy-loaded content by scrolling + """ + print(" [Hook] 📜 Scrolling to load lazy content...") + + # Scroll to bottom + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await page.wait_for_timeout(1000) + + # Scroll to middle + await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 2)") + await page.wait_for_timeout(500) + + # Scroll back to top + await page.evaluate("window.scrollTo(0, 0)") + await page.wait_for_timeout(500) + + print(" [Hook] ✓ Lazy content loaded") + return page + + +async def page_analytics_hook(page, context, **kwargs): + """ + Analytics Hook: Log page metrics before extraction + """ + print(" [Hook] 📊 Collecting page analytics...") + + metrics = await page.evaluate(''' + () => ({ + title: document.title, + images: document.images.length, + links: document.links.length, + scripts: document.scripts.length, + headings: document.querySelectorAll('h1, h2, h3').length, + paragraphs: document.querySelectorAll('p').length + }) + ''') + + print(f" [Hook] 📈 Page: {metrics['title'][:50]}...") + print(f" Links: {metrics['links']}, Images: {metrics['images']}, " + f"Headings: {metrics['headings']}, Paragraphs: {metrics['paragraphs']}") + + return page + + +# ============================================================================ +# DEMO 1: String-Based Hooks (NEW Docker Hooks System) +# ============================================================================ + +def demo_1_string_based_hooks(): + """ + Demonstrate string-based hooks with REST API (part of NEW Docker Hooks System) + """ + print_section( + "DEMO 1: String-Based Hooks (REST API)", + "Part of the NEW Docker Hooks System - hooks as strings" + ) + + # Define hooks as strings + hooks_config = { + "on_page_context_created": """ +async def hook(page, context, **kwargs): + print(" [String Hook] Setting up page context...") + # Block images for performance + await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort()) + await page.set_viewport_size({"width": 1920, "height": 1080}) + return page +""", + + "before_goto": """ +async def hook(page, context, url, **kwargs): + print(f" [String Hook] Navigating to {url[:50]}...") + await page.set_extra_http_headers({ + 'X-Crawl4AI': 'string-based-hooks', + 'X-Demo': 'v0.7.5' + }) + return page +""", + + "before_retrieve_html": """ +async def hook(page, context, **kwargs): + print(" [String Hook] Scrolling page...") + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await page.wait_for_timeout(1000) + return page +""" + } + + # Prepare request payload + payload = { + "urls": [TEST_URLS[0]], + "hooks": { + "code": hooks_config, + "timeout": 30 + }, + "crawler_config": { + "cache_mode": "bypass" + } + } + + print(f"🎯 Target URL: {TEST_URLS[0]}") + print(f"🔧 Configured {len(hooks_config)} string-based hooks") + print(f"📡 Sending request to Docker API...\n") + + try: + start_time = time.time() + response = requests.post(f"{DOCKER_URL}/crawl", json=payload, timeout=60) + execution_time = time.time() - start_time + + if response.status_code == 200: + result = response.json() + + print(f"\n✅ Request successful! (took {execution_time:.2f}s)") + + # Display results + if result.get('results') and result['results'][0].get('success'): + crawl_result = result['results'][0] + html_length = len(crawl_result.get('html', '')) + markdown_length = len(crawl_result.get('markdown', '')) + + print(f"\n📊 Results:") + print(f" • HTML length: {html_length:,} characters") + print(f" • Markdown length: {markdown_length:,} characters") + print(f" • URL: {crawl_result.get('url')}") + + # Check hooks execution + if 'hooks' in result: + hooks_info = result['hooks'] + print(f"\n🎣 Hooks Execution:") + print(f" • Status: {hooks_info['status']['status']}") + print(f" • Attached hooks: {len(hooks_info['status']['attached_hooks'])}") + + if 'summary' in hooks_info: + summary = hooks_info['summary'] + print(f" • Total executions: {summary['total_executions']}") + print(f" • Successful: {summary['successful']}") + print(f" • Success rate: {summary['success_rate']:.1f}%") + else: + print(f"⚠️ Crawl completed but no results") + + else: + print(f"❌ Request failed with status {response.status_code}") + print(f" Error: {response.text[:200]}") + + except requests.exceptions.Timeout: + print("⏰ Request timed out after 60 seconds") + except Exception as e: + print(f"❌ Error: {str(e)}") + + print("\n" + "─" * 70) + print("✓ String-based hooks demo complete\n") + + +# ============================================================================ +# DEMO 2: Function-Based Hooks with hooks_to_string() Utility +# ============================================================================ + +def demo_2_hooks_to_string_utility(): + """ + Demonstrate the new hooks_to_string() utility for converting functions + """ + print_section( + "DEMO 2: hooks_to_string() Utility (NEW! ✨)", + "Convert Python functions to strings for REST API" + ) + + print("📦 Creating hook functions...") + print(" • performance_optimization_hook") + print(" • viewport_setup_hook") + print(" • authentication_headers_hook") + print(" • lazy_loading_handler_hook") + + # Convert function objects to strings using the NEW utility + print("\n🔄 Converting functions to strings with hooks_to_string()...") + + hooks_dict = { + "on_page_context_created": performance_optimization_hook, + "before_goto": authentication_headers_hook, + "before_retrieve_html": lazy_loading_handler_hook, + } + + hooks_as_strings = hooks_to_string(hooks_dict) + + print(f"✅ Successfully converted {len(hooks_as_strings)} functions to strings") + + # Show a preview + print("\n📝 Sample converted hook (first 250 characters):") + print("─" * 70) + sample_hook = list(hooks_as_strings.values())[0] + print(sample_hook[:250] + "...") + print("─" * 70) + + # Use the converted hooks with REST API + print("\n📡 Using converted hooks with REST API...") + + payload = { + "urls": [TEST_URLS[0]], + "hooks": { + "code": hooks_as_strings, + "timeout": 30 + } + } + + try: + start_time = time.time() + response = requests.post(f"{DOCKER_URL}/crawl", json=payload, timeout=60) + execution_time = time.time() - start_time + + if response.status_code == 200: + result = response.json() + print(f"\n✅ Request successful! (took {execution_time:.2f}s)") + + if result.get('results') and result['results'][0].get('success'): + crawl_result = result['results'][0] + print(f" • HTML length: {len(crawl_result.get('html', '')):,} characters") + print(f" • Hooks executed successfully!") + else: + print(f"❌ Request failed: {response.status_code}") + + except Exception as e: + print(f"❌ Error: {str(e)}") + + print("\n💡 Benefits of hooks_to_string():") + print(" ✓ Write hooks as regular Python functions") + print(" ✓ Full IDE support (autocomplete, syntax highlighting)") + print(" ✓ Type checking and linting") + print(" ✓ Easy to test and debug") + print(" ✓ Reusable across projects") + print(" ✓ Works with any REST API client") + + print("\n" + "─" * 70) + print("✓ hooks_to_string() utility demo complete\n") + + +# ============================================================================ +# DEMO 3: Docker Client with Automatic Conversion (RECOMMENDED! 🌟) +# ============================================================================ + +async def demo_3_docker_client_auto_conversion(): + """ + Demonstrate Docker Client with automatic hook conversion (RECOMMENDED) + """ + print_section( + "DEMO 3: Docker Client with Auto-Conversion (RECOMMENDED! 🌟)", + "Pass function objects directly - conversion happens automatically!" + ) + + print("🐳 Initializing Crawl4AI Docker Client...") + client = Crawl4aiDockerClient(base_url=DOCKER_URL) + + print("✅ Client ready!\n") + + # Use our reusable hook library - just pass the function objects! + print("📚 Using reusable hook library:") + print(" • performance_optimization_hook") + print(" • viewport_setup_hook") + print(" • authentication_headers_hook") + print(" • lazy_loading_handler_hook") + print(" • page_analytics_hook") + + print("\n🎯 Target URL: " + TEST_URLS[1]) + print("🚀 Starting crawl with automatic hook conversion...\n") + + try: + start_time = time.time() + + # Pass function objects directly - NO manual conversion needed! ✨ + results = await client.crawl( + urls=[TEST_URLS[0]], + hooks={ + "on_page_context_created": performance_optimization_hook, + "before_goto": authentication_headers_hook, + "before_retrieve_html": lazy_loading_handler_hook, + "before_return_html": page_analytics_hook, + }, + hooks_timeout=30 + ) + + execution_time = time.time() - start_time + + print(f"\n✅ Crawl completed! (took {execution_time:.2f}s)\n") + + # Display results + if results and results.success: + result = results + print(f"📊 Results:") + print(f" • URL: {result.url}") + print(f" • Success: {result.success}") + print(f" • HTML length: {len(result.html):,} characters") + print(f" • Markdown length: {len(result.markdown):,} characters") + + # Show metadata + if result.metadata: + print(f"\n📋 Metadata:") + print(f" • Title: {result.metadata.get('title', 'N/A')}") + print(f" • Description: {result.metadata.get('description', 'N/A')}") + + # Show links + if result.links: + internal_count = len(result.links.get('internal', [])) + external_count = len(result.links.get('external', [])) + print(f"\n🔗 Links Found:") + print(f" • Internal: {internal_count}") + print(f" • External: {external_count}") + else: + print(f"⚠️ Crawl completed but no successful results") + if results: + print(f" Error: {results.error_message}") + + except Exception as e: + print(f"❌ Error: {str(e)}") + import traceback + traceback.print_exc() + + print("\n🌟 Why Docker Client is RECOMMENDED:") + print(" ✓ Automatic function-to-string conversion") + print(" ✓ No manual hooks_to_string() calls needed") + print(" ✓ Cleaner, more Pythonic code") + print(" ✓ Full type hints and IDE support") + print(" ✓ Built-in error handling") + print(" ✓ Async/await support") + + print("\n" + "─" * 70) + print("✓ Docker Client auto-conversion demo complete\n") + + +# ============================================================================ +# DEMO 4: Advanced Use Case - Complete Hook Pipeline +# ============================================================================ + +async def demo_4_complete_hook_pipeline(): + """ + Demonstrate a complete hook pipeline using all 8 hook points + """ + print_section( + "DEMO 4: Complete Hook Pipeline", + "Using all 8 available hook points for comprehensive control" + ) + + # Define all 8 hooks + async def on_browser_created_hook(browser, **kwargs): + """Hook 1: Called after browser is created""" + print(" [Pipeline] 1/8 Browser created") + return browser + + async def on_page_context_created_hook(page, context, **kwargs): + """Hook 2: Called after page context is created""" + print(" [Pipeline] 2/8 Page context created - setting up...") + await page.set_viewport_size({"width": 1920, "height": 1080}) + return page + + async def on_user_agent_updated_hook(page, context, user_agent, **kwargs): + """Hook 3: Called when user agent is updated""" + print(f" [Pipeline] 3/8 User agent updated: {user_agent[:50]}...") + return page + + async def before_goto_hook(page, context, url, **kwargs): + """Hook 4: Called before navigating to URL""" + print(f" [Pipeline] 4/8 Before navigation to: {url[:60]}...") + return page + + async def after_goto_hook(page, context, url, response, **kwargs): + """Hook 5: Called after navigation completes""" + print(f" [Pipeline] 5/8 After navigation - Status: {response.status if response else 'N/A'}") + await page.wait_for_timeout(1000) + return page + + async def on_execution_started_hook(page, context, **kwargs): + """Hook 6: Called when JavaScript execution starts""" + print(" [Pipeline] 6/8 JavaScript execution started") + return page + + async def before_retrieve_html_hook(page, context, **kwargs): + """Hook 7: Called before retrieving HTML""" + print(" [Pipeline] 7/8 Before HTML retrieval - scrolling...") + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + return page + + async def before_return_html_hook(page, context, html, **kwargs): + """Hook 8: Called before returning HTML""" + print(f" [Pipeline] 8/8 Before return - HTML length: {len(html):,} chars") + return page + + print("🎯 Target URL: " + TEST_URLS[0]) + print("🔧 Configured ALL 8 hook points for complete pipeline control\n") + + client = Crawl4aiDockerClient(base_url=DOCKER_URL) + + try: + print("🚀 Starting complete pipeline crawl...\n") + start_time = time.time() + + results = await client.crawl( + urls=[TEST_URLS[0]], + hooks={ + "on_browser_created": on_browser_created_hook, + "on_page_context_created": on_page_context_created_hook, + "on_user_agent_updated": on_user_agent_updated_hook, + "before_goto": before_goto_hook, + "after_goto": after_goto_hook, + "on_execution_started": on_execution_started_hook, + "before_retrieve_html": before_retrieve_html_hook, + "before_return_html": before_return_html_hook, + }, + hooks_timeout=45 + ) + + execution_time = time.time() - start_time + + if results and results.success: + print(f"\n✅ Complete pipeline executed successfully! (took {execution_time:.2f}s)") + print(f" • All 8 hooks executed in sequence") + print(f" • HTML length: {len(results.html):,} characters") + else: + print(f"⚠️ Pipeline completed with warnings") + + except Exception as e: + print(f"❌ Error: {str(e)}") + + print("\n📚 Available Hook Points:") + print(" 1. on_browser_created - Browser initialization") + print(" 2. on_page_context_created - Page context setup") + print(" 3. on_user_agent_updated - User agent configuration") + print(" 4. before_goto - Pre-navigation setup") + print(" 5. after_goto - Post-navigation processing") + print(" 6. on_execution_started - JavaScript execution start") + print(" 7. before_retrieve_html - Pre-extraction processing") + print(" 8. before_return_html - Final HTML processing") + + print("\n" + "─" * 70) + print("✓ Complete hook pipeline demo complete\n") + + +# ============================================================================ +# MAIN EXECUTION +# ============================================================================ + +async def main(): + """ + Run all demonstrations + """ + print("\n" + "=" * 70) + print(" 🚀 Crawl4AI v0.7.5 - Docker Hooks Complete Demonstration") + print("=" * 70) + + # Check Docker service + print("\n🔍 Checking Docker service status...") + if not check_docker_service(): + print("❌ Docker service is not running!") + print("\n📋 To start the Docker service:") + print(" docker run -p 11235:11235 unclecode/crawl4ai:latest") + print("\nPlease start the service and run this demo again.") + return + + print("✅ Docker service is running!\n") + + # Run all demos + demos = [ + ("String-Based Hooks (REST API)", demo_1_string_based_hooks, False), + ("hooks_to_string() Utility", demo_2_hooks_to_string_utility, False), + ("Docker Client Auto-Conversion", demo_3_docker_client_auto_conversion, True), + ("Complete Hook Pipeline", demo_4_complete_hook_pipeline, True), + ] + + for i, (name, demo_func, is_async) in enumerate(demos, 1): + print(f"\n{'🔷' * 35}") + print(f"Starting Demo {i}/{len(demos)}: {name}") + print(f"{'🔷' * 35}\n") + + try: + if is_async: + await demo_func() + else: + demo_func() + + print(f"✅ Demo {i} completed successfully!") + + # Pause between demos (except the last one) + if i < len(demos): + print("\n⏸️ Press Enter to continue to next demo...") + input() + + except KeyboardInterrupt: + print(f"\n⏹️ Demo interrupted by user") + break + except Exception as e: + print(f"\n❌ Demo {i} failed: {str(e)}") + import traceback + traceback.print_exc() + print("\nContinuing to next demo...\n") + continue + + # Final summary + # print("\n" + "=" * 70) + # print(" 🎉 All Demonstrations Complete!") + # print("=" * 70) + + # print("\n📊 Summary of v0.7.5 Docker Hooks System:") + # print("\n🆕 COMPLETELY NEW FEATURE in v0.7.5:") + # print(" The Docker Hooks System lets you customize the crawling pipeline") + # print(" with user-provided Python functions at 8 strategic points.") + + # print("\n✨ Three Ways to Use Docker Hooks (All NEW!):") + # print(" 1. String-based - Write hooks as strings for REST API") + # print(" 2. hooks_to_string() - Convert Python functions to strings") + # print(" 3. Docker Client - Automatic conversion (RECOMMENDED)") + + # print("\n💡 Key Benefits:") + # print(" ✓ Full IDE support (autocomplete, syntax highlighting)") + # print(" ✓ Type checking and linting") + # print(" ✓ Easy to test and debug") + # print(" ✓ Reusable across projects") + # print(" ✓ Complete pipeline control") + + # print("\n🎯 8 Hook Points Available:") + # print(" • on_browser_created, on_page_context_created") + # print(" • on_user_agent_updated, before_goto, after_goto") + # print(" • on_execution_started, before_retrieve_html, before_return_html") + + # print("\n📚 Resources:") + # print(" • Docs: https://docs.crawl4ai.com") + # print(" • GitHub: https://github.com/unclecode/crawl4ai") + # print(" • Discord: https://discord.gg/jP8KfhDhyN") + + # print("\n" + "=" * 70) + # print(" Happy Crawling with v0.7.5! 🕷️") + # print("=" * 70 + "\n") + + +if __name__ == "__main__": + print("\n🎬 Starting Crawl4AI v0.7.5 Docker Hooks Demonstration...") + print("Press Ctrl+C anytime to exit\n") + + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\n\n👋 Demo stopped by user. Thanks for exploring Crawl4AI v0.7.5!") + except Exception as e: + print(f"\n\n❌ Demo error: {str(e)}") + import traceback + traceback.print_exc() diff --git a/docs/releases_review/v0.7.5_video_walkthrough.ipynb b/docs/releases_review/v0.7.5_video_walkthrough.ipynb new file mode 100644 index 00000000..a57de4c9 --- /dev/null +++ b/docs/releases_review/v0.7.5_video_walkthrough.ipynb @@ -0,0 +1,1516 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 🚀 Crawl4AI v0.7.5 - Complete Feature Walkthrough\n", + "\n", + "Welcome to Crawl4AI v0.7.5! This notebook demonstrates all the new features introduced in this release.\n", + "\n", + "## 📋 What's New in v0.7.5\n", + "\n", + "1. **🔧 Docker Hooks System** - NEW! Complete pipeline customization with user-provided Python functions\n", + "2. **🤖 Enhanced LLM Integration** - Custom providers with temperature control\n", + "3. **🔒 HTTPS Preservation** - Secure internal link handling\n", + "4. **🛠️ Multiple Bug Fixes** - Community-reported issues resolved\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 📦 Setup and Installation\n", + "\n", + "First, let's make sure we have the latest version installed:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Crawl4AI v0.7.5 ready!\n" + ] + } + ], + "source": [ + "# # Install or upgrade to v0.7.5\n", + "# !pip install -U crawl4ai==0.7.5 --quiet\n", + "\n", + "# Import required modules\n", + "import asyncio\n", + "import nest_asyncio\n", + "nest_asyncio.apply() # For Jupyter compatibility\n", + "\n", + "from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode\n", + "from crawl4ai import FilterChain, URLPatternFilter, BFSDeepCrawlStrategy\n", + "from crawl4ai import hooks_to_string\n", + "\n", + "print(\"✅ Crawl4AI v0.7.5 ready!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 🔒 Feature 1: HTTPS Preservation for Internal Links\n", + "\n", + "### Problem\n", + "When crawling HTTPS sites, internal links sometimes get downgraded to HTTP, breaking authentication and causing security warnings.\n", + "\n", + "### Solution \n", + "The new `preserve_https_for_internal_links=True` parameter maintains HTTPS protocol for all internal links." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🔒 Testing HTTPS Preservation\n", + "\n", + "============================================================\n" + ] + }, + { + "data": { + "text/html": [ + "
[INIT].... → Crawl4AI 0.7.5 \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;36m[\u001b[0m\u001b[36mINIT\u001b[0m\u001b[1;36m]\u001b[0m\u001b[36m...\u001b[0m\u001b[36m. → Crawl4AI \u001b[0m\u001b[1;36m0.7\u001b[0m\u001b[36m.\u001b[0m\u001b[1;36m5\u001b[0m\u001b[36m \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[FETCH]... ↓ https://quotes.toscrape.com                                                                          |\n",
+       "✓ | ⏱: 1.98s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mFETCH\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m...\u001b[0m\u001b[32m ↓ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m1.\u001b[0m\u001b[32m98s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[SCRAPE].. ◆ https://quotes.toscrape.com                                                                          |\n",
+       "✓ | ⏱: 0.01s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mSCRAPE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m.. ◆ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m01s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[COMPLETE] ● https://quotes.toscrape.com                                                                          |\n",
+       "✓ | ⏱: 2.00s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mCOMPLETE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m ● \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m2.\u001b[0m\u001b[32m00s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[FETCH]... ↓ https://quotes.toscrape.com                                                                          |\n",
+       "✓ | ⏱: 0.72s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mFETCH\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m...\u001b[0m\u001b[32m ↓ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m72s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[SCRAPE].. ◆ https://quotes.toscrape.com                                                                          |\n",
+       "✓ | ⏱: 0.01s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mSCRAPE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m.. ◆ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m01s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[COMPLETE] ● https://quotes.toscrape.com                                                                          |\n",
+       "✓ | ⏱: 0.73s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mCOMPLETE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m ● \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m73s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[FETCH]... ↓ https://quotes.toscrape.com/login                                                                    |\n",
+       "✓ | ⏱: 0.83s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mFETCH\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m...\u001b[0m\u001b[32m ↓ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/login\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m83s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[SCRAPE].. ◆ https://quotes.toscrape.com/login                                                                    |\n",
+       "✓ | ⏱: 0.00s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mSCRAPE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m.. ◆ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/login\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m00s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[COMPLETE] ● https://quotes.toscrape.com/login                                                                    |\n",
+       "✓ | ⏱: 0.83s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mCOMPLETE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m ● \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/login\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m83s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[FETCH]... ↓ https://quotes.toscrape.com/tag/change/page/1                                                        |\n",
+       "✓ | ⏱: 1.11s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mFETCH\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m...\u001b[0m\u001b[32m ↓ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/tag/change/page/1\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m1.\u001b[0m\u001b[32m11s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[SCRAPE].. ◆ https://quotes.toscrape.com/tag/change/page/1                                                        |\n",
+       "✓ | ⏱: 0.00s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mSCRAPE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m.. ◆ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/tag/change/page/1\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m00s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[COMPLETE] ● https://quotes.toscrape.com/tag/change/page/1                                                        |\n",
+       "✓ | ⏱: 1.12s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mCOMPLETE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m ● \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/tag/change/page/1\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m1.\u001b[0m\u001b[32m12s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[FETCH]... ↓ https://quotes.toscrape.com/author/Albert-Einstein                                                   |\n",
+       "✓ | ⏱: 1.32s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mFETCH\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m...\u001b[0m\u001b[32m ↓ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/author/Albert-Einstein\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m1.\u001b[0m\u001b[32m32s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[SCRAPE].. ◆ https://quotes.toscrape.com/author/Albert-Einstein                                                   |\n",
+       "✓ | ⏱: 0.00s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mSCRAPE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m.. ◆ \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/author/Albert-Einstein\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m00s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[COMPLETE] ● https://quotes.toscrape.com/author/Albert-Einstein                                                   |\n",
+       "✓ | ⏱: 1.33s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mCOMPLETE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m ● \u001b[0m\u001b[4;32mhttps://quotes.toscrape.com/author/Albert-Einstein\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m1.\u001b[0m\u001b[32m33s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "📊 Results:\n", + " Pages crawled: 5\n", + " Total internal links (from first page): 47\n", + " HTTPS links: 47 ✅\n", + " HTTP links: 0 \n", + " HTTPS preservation rate: 100.0%\n", + "\n", + "🔗 Sample HTTPS-preserved links:\n", + " → https://quotes.toscrape.com/\n", + " → https://quotes.toscrape.com/login\n", + " → https://quotes.toscrape.com/author/Albert-Einstein\n", + " → https://quotes.toscrape.com/tag/change/page/1\n", + " → https://quotes.toscrape.com/tag/deep-thoughts/page/1\n", + "\n", + "============================================================\n", + "✅ HTTPS Preservation Demo Complete!\n", + "\n" + ] + } + ], + "source": [ + "async def demo_https_preservation():\n", + " \"\"\"\n", + " Demonstrate HTTPS preservation with deep crawling\n", + " \"\"\"\n", + " print(\"🔒 Testing HTTPS Preservation\\n\")\n", + " print(\"=\" * 60)\n", + " \n", + " # Setup URL filter for quotes.toscrape.com\n", + " url_filter = URLPatternFilter(\n", + " patterns=[r\"^(https:\\/\\/)?quotes\\.toscrape\\.com(\\/.*)?$\"]\n", + " )\n", + " \n", + " # Configure crawler with HTTPS preservation\n", + " config = CrawlerRunConfig(\n", + " exclude_external_links=True,\n", + " preserve_https_for_internal_links=True, # 🆕 NEW in v0.7.5\n", + " cache_mode=CacheMode.BYPASS,\n", + " deep_crawl_strategy=BFSDeepCrawlStrategy(\n", + " max_depth=2,\n", + " max_pages=5,\n", + " filter_chain=FilterChain([url_filter])\n", + " )\n", + " )\n", + " \n", + " async with AsyncWebCrawler() as crawler:\n", + " # With deep_crawl_strategy, arun() returns a list of CrawlResult objects\n", + " results = await crawler.arun(\n", + " url=\"https://quotes.toscrape.com\",\n", + " config=config\n", + " )\n", + " \n", + " # Analyze the first result\n", + " if results and len(results) > 0:\n", + " first_result = results[0]\n", + " internal_links = [link['href'] for link in first_result.links['internal']]\n", + " \n", + " # Check HTTPS preservation\n", + " https_links = [link for link in internal_links if link.startswith('https://')]\n", + " http_links = [link for link in internal_links if link.startswith('http://') and not link.startswith('https://')]\n", + " \n", + " print(f\"\\n📊 Results:\")\n", + " print(f\" Pages crawled: {len(results)}\")\n", + " print(f\" Total internal links (from first page): {len(internal_links)}\")\n", + " print(f\" HTTPS links: {len(https_links)} ✅\")\n", + " print(f\" HTTP links: {len(http_links)} {'⚠️' if http_links else ''}\")\n", + " if internal_links:\n", + " print(f\" HTTPS preservation rate: {len(https_links)/len(internal_links)*100:.1f}%\")\n", + " \n", + " print(f\"\\n🔗 Sample HTTPS-preserved links:\")\n", + " for link in https_links[:5]:\n", + " print(f\" → {link}\")\n", + " else:\n", + " print(f\"\\n⚠️ No results returned\")\n", + " \n", + " print(\"\\n\" + \"=\" * 60)\n", + " print(\"✅ HTTPS Preservation Demo Complete!\\n\")\n", + "\n", + "# Run the demo\n", + "await demo_https_preservation()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 🤖 Feature 2: Enhanced LLM Integration\n", + "\n", + "### What's New\n", + "- Custom `temperature` parameter for creativity control\n", + "- `base_url` for custom API endpoints\n", + "- Better multi-provider support\n", + "\n", + "### Example with Custom Temperature" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🤖 Testing Enhanced LLM Integration\n", + "\n", + "============================================================\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/k0/7502j87n0_q4f9g82c0w8ks80000gn/T/ipykernel_15029/173393508.py:47: PydanticDeprecatedSince20: The `schema` method is deprecated; use `model_json_schema` instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.12/migration/\n", + " schema=Article.schema(),\n" + ] + }, + { + "data": { + "text/html": [ + "
[INIT].... → Crawl4AI 0.7.5 \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;36m[\u001b[0m\u001b[36mINIT\u001b[0m\u001b[1;36m]\u001b[0m\u001b[36m...\u001b[0m\u001b[36m. → Crawl4AI \u001b[0m\u001b[1;36m0.7\u001b[0m\u001b[36m.\u001b[0m\u001b[1;36m5\u001b[0m\u001b[36m \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[FETCH]... ↓ https://en.wikipedia.org/wiki/Artificial_intelligence                                                |\n",
+       "✓ | ⏱: 3.05s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mFETCH\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m...\u001b[0m\u001b[32m ↓ \u001b[0m\u001b[4;32mhttps://en.wikipedia.org/wiki/Artificial_intelligence\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m3.\u001b[0m\u001b[32m05s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[SCRAPE].. ◆ https://en.wikipedia.org/wiki/Artificial_intelligence                                                |\n",
+       "✓ | ⏱: 0.63s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mSCRAPE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m.. ◆ \u001b[0m\u001b[4;32mhttps://en.wikipedia.org/wiki/Artificial_intelligence\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m63s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[EXTRACT]. ■ https://en.wikipedia.org/wiki/Artificial_intelligence                                                |\n",
+       "✓ | ⏱: 20.74s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mEXTRACT\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m. ■ \u001b[0m\u001b[4;32mhttps://en.wikipedia.org/wiki/Artificial_intelligence\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m20.\u001b[0m\u001b[32m74s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[COMPLETE] ● https://en.wikipedia.org/wiki/Artificial_intelligence                                                |\n",
+       "✓ | ⏱: 24.42s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mCOMPLETE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m ● \u001b[0m\u001b[4;32mhttps://en.wikipedia.org/wiki/Artificial_intelligence\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m24.\u001b[0m\u001b[32m42s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "✅ LLM Extraction Successful!\n", + "\n", + "📄 Extracted Content:\n", + "[\n", + " {\n", + " \"title\": \"Artificial intelligence\",\n", + " \"summary\": \"Artificial intelligence (AI) refers to the simulation of human intelligence in machines that are programmed to think and learn like humans. AI can be applied in various fields and has numerous applications, including health, finance, and military.\",\n", + " \"main_topics\": [\n", + " \"Goals\",\n", + " \"Techniques\",\n", + " \"Applications\",\n", + " \"Ethics\",\n", + " \"History\",\n", + " \"Philosophy\",\n", + " \"Future\",\n", + " \"In fiction\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial intelligence\",\n", + " \"summary\": \"The article discusses artificial intelligence (AI), its various techniques, applications, and advancements, particularly focusing on machine learning, deep learning, and neural networks. It highlights the evolution of AI technologies, including generative pre-trained transformers (GPT), and their impact on fields such as healthcare, gaming, and mathematics.\",\n", + " \"main_topics\": [\n", + " \"Classifiers and pattern matching\",\n", + " \"Artificial neural networks\",\n", + " \"Deep learning\",\n", + " \"Generative pre-trained transformers (GPT)\",\n", + " \"Hardware and software for AI\",\n", + " \"Applications of AI\",\n", + " \"AI in healthcare\",\n", + " \"AI in games\",\n", + " \"AI in mathematics\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial intelligence\",\n", + " \"summary\": \"Artificial intelligence (AI) is the capability of computational systems to perform tasks typically associated with human intelligence, such as learning, reasoning, problem-solving, perception, and decision-making. It is a field of research in computer science that develops methods and software enabling machines to perceive their environment and take actions to achieve defined goals. AI has seen significant advancements and applications in various domains, including web search engines, recommendation systems, virtual assistants, and autonomous vehicles, among others.\",\n", + " \"main_topics\": [\n", + " \"Goals\",\n", + " \"Reasoning and problem-solving\",\n", + " \"Knowledge representation\",\n", + " \"Planning and decision-making\",\n", + " \"Learning\",\n", + " \"Applications\",\n", + " \"Philosophy\",\n", + " \"History\",\n", + " \"Controversies\",\n", + " \"Ethics\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial intelligence\",\n", + " \"summary\": \"The article discusses artificial intelligence (AI), its various techniques, and applications. It covers the foundational concepts of AI, including machine learning, natural language processing, perception, social intelligence, and general intelligence. The article also highlights the methods used in AI research, such as search and optimization, logic, probabilistic methods, and classifiers.\",\n", + " \"main_topics\": [\n", + " \"Markov decision processes\",\n", + " \"Machine learning\",\n", + " \"Supervised learning\",\n", + " \"Unsupervised learning\",\n", + " \"Reinforcement learning\",\n", + " \"Transfer learning\",\n", + " \"Deep learning\",\n", + " \"Natural language processing\",\n", + " \"Machine perception\",\n", + " \"Social intelligence\",\n", + " \"Artificial general intelligence\",\n", + " \"Search and optimization\",\n", + " \"Logic\",\n", + " \"Probabilistic methods\",\n", + " \"Classifiers and statistical learning methods\",\n", + " \"Artificial neural networks\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial Intelligence\",\n", + " \"summary\": \"The article discusses the complexities and challenges associated with artificial intelligence (AI), particularly focusing on issues of bias, fairness, transparency, and the potential risks posed by AI technologies. It highlights the ethical implications of AI systems, the lack of diversity among AI developers, and the existential risks associated with advanced AI.\",\n", + " \"main_topics\": [\n", + " \"Bias and fairness in AI\",\n", + " \"Lack of transparency in AI systems\",\n", + " \"Weaponization of AI\",\n", + " \"Technological unemployment due to AI\",\n", + " \"Existential risk from AI\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial intelligence\",\n", + " \"summary\": \"The article discusses the advancements and applications of artificial intelligence (AI) across various fields, including mathematics, finance, military, generative AI, and more. It highlights the capabilities of AI models, their limitations, and the ethical considerations surrounding their use.\",\n", + " \"main_topics\": [\n", + " \"Mathematics\",\n", + " \"Finance\",\n", + " \"Military applications\",\n", + " \"Generative AI\",\n", + " \"AI agents\",\n", + " \"Web search\",\n", + " \"Sexuality\",\n", + " \"Industry-specific tasks\",\n", + " \"Ethics\",\n", + " \"Privacy and copyright\",\n", + " \"Dominance by tech giants\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial Intelligence\",\n", + " \"summary\": \"The article discusses various aspects of artificial intelligence (AI), including its impact on privacy, copyright issues, environmental concerns, misinformation, and algorithmic bias. It highlights the dominance of big tech companies in the AI landscape and the increasing power demands of AI technologies.\",\n", + " \"main_topics\": [\n", + " \"Privacy and Fairness\",\n", + " \"Generative AI and Copyright\",\n", + " \"Dominance by Tech Giants\",\n", + " \"Power Needs and Environmental Impacts\",\n", + " \"Misinformation\",\n", + " \"Algorithmic Bias and Fairness\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial intelligence\",\n", + " \"summary\": \"The article discusses the mixed opinions among experts regarding the risks associated with artificial intelligence (AI), particularly concerning superintelligent AI. It highlights concerns from notable figures in the field about existential risks, the importance of establishing safety guidelines, and the ongoing debate between pessimistic and optimistic views on AI's future impact. The article also covers ethical considerations, open-source developments, regulatory efforts, and the historical context of AI research.\",\n", + " \"main_topics\": [\n", + " \"Expert opinions on AI risks\",\n", + " \"Existential risk from superintelligent AI\",\n", + " \"Ethical machines and alignment\",\n", + " \"Open-source AI\",\n", + " \"Regulation of artificial intelligence\",\n", + " \"History of artificial intelligence\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial Intelligence\",\n", + " \"summary\": \"The article discusses the history, development, and various approaches to artificial intelligence (AI), highlighting key milestones, challenges, and philosophical debates surrounding the field. It covers the evolution from early optimism and funding cuts to the resurgence of interest through expert systems and deep learning, as well as the implications of AI advancements on society.\",\n", + " \"main_topics\": [\n", + " \"History of AI\",\n", + " \"AI winter\",\n", + " \"Expert systems\",\n", + " \"Deep learning\",\n", + " \"Artificial general intelligence (AGI)\",\n", + " \"Philosophy of AI\",\n", + " \"Defining artificial intelligence\",\n", + " \"Evaluating approaches to AI\",\n", + " \"Symbolic AI vs. sub-symbolic AI\",\n", + " \"Narrow AI vs. general AI\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial intelligence\",\n", + " \"summary\": \"Artificial intelligence (AI) is the simulation of human intelligence processes by machines, especially computer systems. It encompasses various subfields including machine learning, natural language processing, and robotics, and aims to create systems that can perform tasks that typically require human intelligence.\",\n", + " \"main_topics\": [\n", + " \"Organoid intelligence\",\n", + " \"Robotic process automation\",\n", + " \"Wetware computer\",\n", + " \"DARWIN EU\",\n", + " \"Artificial intelligence in Wikimedia projects\",\n", + " \"AI-generated content on Wikipedia\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial intelligence\",\n", + " \"summary\": \"The article discusses the field of artificial intelligence (AI), exploring its various branches, methodologies, and philosophical implications. It highlights the ongoing debates within the AI community regarding the pursuit of general versus narrow AI, the nature of consciousness in machines, and the ethical considerations surrounding AI rights and welfare.\",\n", + " \"main_topics\": [\n", + " \"Soft vs. hard computing\",\n", + " \"Narrow vs. general AI\",\n", + " \"Philosophy of artificial intelligence\",\n", + " \"Consciousness\",\n", + " \"Computationalism and functionalism\",\n", + " \"AI welfare and rights\",\n", + " \"Superintelligence and the singularity\",\n", + " \"Transhumanism\",\n", + " \"Artificial intelligence in fiction\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial Intelligence\",\n", + " \"summary\": \"The article discusses the field of artificial intelligence (AI), covering its definitions, history, methodologies, and applications. It explores various aspects of AI, including machine learning, natural language processing, and robotics, as well as the challenges and ethical considerations associated with AI technologies.\",\n", + " \"main_topics\": [\n", + " \"Definitions of AI\",\n", + " \"History of AI\",\n", + " \"Machine Learning\",\n", + " \"Natural Language Processing\",\n", + " \"Robotics\",\n", + " \"Ethical Considerations\",\n", + " \"Applications of AI\",\n", + " \"AI Methodologies\",\n", + " \"Challenges in AI\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial Intelligence\",\n", + " \"summary\": \"The article discusses the advancements and implications of artificial intelligence (AI), particularly focusing on generative AI and its impact across various sectors including healthcare, finance, entertainment, and environmental concerns.\",\n", + " \"main_topics\": [\n", + " \"Generative AI in software development\",\n", + " \"AI in healthcare\",\n", + " \"AI in financial services\",\n", + " \"Impact of AI on Hollywood and entertainment\",\n", + " \"AI and environmental issues\",\n", + " \"AI's role in creativity\",\n", + " \"AI in search technologies\",\n", + " \"AI's energy consumption\",\n", + " \"AI and societal implications\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial Intelligence\",\n", + " \"summary\": \"The article discusses the concept of artificial intelligence (AI), its development, applications, and the ethical implications surrounding its use. It highlights the advancements in AI technology, including synthetic media and computational capitalism, and addresses concerns regarding misinformation and media manipulation through AI tools.\",\n", + " \"main_topics\": [\n", + " \"Definition of Artificial Intelligence\",\n", + " \"Advancements in AI technology\",\n", + " \"Synthetic media and computational capitalism\",\n", + " \"Ethical implications of AI\",\n", + " \"Misinformation and media manipulation\",\n", + " \"AI in surveillance and security\",\n", + " \"AI's impact on employment\",\n", + " \"Global regulatory frameworks for AI\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial Intelligence\",\n", + " \"summary\": \"The article discusses the concept of artificial intelligence (AI), its applications, advancements, and implications across various fields, including healthcare, programming, and national security. It highlights the evolution of AI technologies, notable achievements, and the ongoing debates surrounding ethical considerations and the future of AI.\",\n", + " \"main_topics\": [\n", + " \"Definition of Artificial Intelligence\",\n", + " \"Applications in Healthcare\",\n", + " \"AI Programming Languages\",\n", + " \"Ethical Considerations\",\n", + " \"AI in National Security\",\n", + " \"Generative AI\",\n", + " \"Recent Advancements in AI Technologies\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial Intelligence\",\n", + " \"summary\": \"The article discusses the field of artificial intelligence (AI), its development, applications, and the ethical considerations surrounding its use. It highlights the advancements in AI technologies, the impact on various sectors, and the ongoing debates regarding the implications of AI on society.\",\n", + " \"main_topics\": [\n", + " \"Definition of Artificial Intelligence\",\n", + " \"History and Development of AI\",\n", + " \"Applications of AI\",\n", + " \"Ethical Considerations in AI\",\n", + " \"Impact of AI on Employment\",\n", + " \"Governance and Regulation of AI\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial Intelligence\",\n", + " \"summary\": \"The article provides an overview of artificial intelligence (AI), its history, development, and various applications. It discusses the evolution of AI from its inception to its current state, highlighting key milestones and influential figures in the field. The article also addresses the philosophical implications of AI, its impact on society, and the ongoing debates surrounding its future.\",\n", + " \"main_topics\": [\n", + " \"History of AI\",\n", + " \"Key figures in AI development\",\n", + " \"Philosophical implications of AI\",\n", + " \"Applications of AI\",\n", + " \"Current trends in AI\",\n", + " \"Ethical considerations in AI\",\n", + " \"Future of AI\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial Intelligence\",\n", + " \"summary\": \"The article discusses various aspects of artificial intelligence (AI), including its implications, challenges, and the need for regulatory frameworks to ensure ethical use. It highlights the perspectives of experts on the responsibilities of tech companies and governments in managing AI technologies.\",\n", + " \"main_topics\": [\n", + " \"Ethical implications of AI\",\n", + " \"Regulatory frameworks for AI\",\n", + " \"Transparency in AI systems\",\n", + " \"Compensation for data usage\",\n", + " \"Professional licensing for AI engineers\",\n", + " \"Limitations of natural language processing\",\n", + " \"AI in media and misinformation\",\n", + " \"AI technologies and their reliability\",\n", + " \"Generative AI and its understanding\",\n", + " \"AI applications in various fields\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial Intelligence\",\n", + " \"summary\": \"The article discusses the field of artificial intelligence (AI), its history, development, and various applications. It highlights the concerns and ethical considerations surrounding AI, as well as the potential impact on society and the economy.\",\n", + " \"main_topics\": [\n", + " \"History of AI\",\n", + " \"Applications of AI\",\n", + " \"Ethical considerations\",\n", + " \"Impact on society\",\n", + " \"Machine learning\",\n", + " \"Regulation of AI\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial Intelligence\",\n", + " \"summary\": \"The article discusses the field of artificial intelligence (AI), covering its history, development, and various applications. It highlights the advancements in AI technologies, the ethical implications, and the ongoing debates surrounding AI's impact on society.\",\n", + " \"main_topics\": [\n", + " \"History of AI\",\n", + " \"Development of AI technologies\",\n", + " \"Applications of AI\",\n", + " \"Ethical implications of AI\",\n", + " \"Debates on AI's societal impact\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial intelligence\",\n", + " \"summary\": \"Artificial intelligence (AI) is the simulation of human intelligence processes by machines, especially computer systems. These processes include learning, reasoning, and self-correction. AI applications include expert systems, natural language processing, speech recognition, and machine vision.\",\n", + " \"main_topics\": [\n", + " \"Neural networks\",\n", + " \"Deep learning\",\n", + " \"Language models\",\n", + " \"Artificial general intelligence (AGI)\",\n", + " \"Computer vision\",\n", + " \"Speech recognition\",\n", + " \"Natural language processing\",\n", + " \"Robotics\",\n", + " \"Philosophy of artificial intelligence\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial intelligence\",\n", + " \"summary\": \"Artificial intelligence (AI) is the simulation of human intelligence processes by machines, especially computer systems. These processes include learning, reasoning, and self-correction.\",\n", + " \"main_topics\": [\n", + " \"Definition of AI\",\n", + " \"Processes involved in AI\",\n", + " \"Applications of AI\",\n", + " \"Types of AI\",\n", + " \"Ethical considerations in AI\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial Intelligence\",\n", + " \"summary\": \"Artificial intelligence (AI) is the simulation of human intelligence processes by machines, especially computer systems. These processes include learning, reasoning, and self-correction. AI applications include expert systems, natural language processing, speech recognition, and machine vision.\",\n", + " \"main_topics\": [\n", + " \"Natural language processing\",\n", + " \"Knowledge representation and reasoning\",\n", + " \"Computer vision\",\n", + " \"Automated planning and scheduling\",\n", + " \"Search methodology\",\n", + " \"Control method\",\n", + " \"Philosophy of artificial intelligence\",\n", + " \"Distributed artificial intelligence\",\n", + " \"Machine learning\"\n", + " ],\n", + " \"error\": false\n", + " },\n", + " {\n", + " \"title\": \"Artificial intelligence\",\n", + " \"summary\": \"Artificial intelligence (AI) is intelligence demonstrated by machines, in contrast to the natural intelligence displayed by humans and animals. Leading AI textbooks define the field as the study of \\\"intelligent agents\\\": any device that perceives its environment and takes actions that maximize its chance of successfully achieving its goals. Colloquially, the term \\\"artificial intelligence\\\" is often used to describe machines (or computers) that mimic \\\"cognitive\\\" functions that humans associate with the human mind, such as \\\"learning\\\" and \\\"problem-solving\\\".\",\n", + " \"main_topics\": [\n", + " \"Automation\",\n", + " \"Ethics of technology\",\n", + " \"AI alignment\",\n", + " \"AI safety\",\n", + " \"Technological singularity\",\n", + " \"Machine ethics\",\n", + " \"Existential risk from artificial intelligence\",\n", + " \"Artificial general intelligence\",\n", + " \"AI takeover\",\n", + " \"AI capability control\"\n", + " ],\n", + " \"error\": false\n", + " }\n", + "]\n", + "\n", + "============================================================\n", + "✅ Enhanced LLM Demo Complete!\n", + "\n" + ] + } + ], + "source": [ + "from crawl4ai import LLMExtractionStrategy, LLMConfig\n", + "from pydantic import BaseModel, Field\n", + "import os\n", + "\n", + "# Define extraction schema\n", + "class Article(BaseModel):\n", + " title: str = Field(description=\"Article title\")\n", + " summary: str = Field(description=\"Brief summary of the article\")\n", + " main_topics: list[str] = Field(description=\"List of main topics covered\")\n", + "\n", + "async def demo_enhanced_llm():\n", + " \"\"\"\n", + " Demonstrate enhanced LLM integration with custom temperature\n", + " \"\"\"\n", + " print(\"🤖 Testing Enhanced LLM Integration\\n\")\n", + " print(\"=\" * 60)\n", + " \n", + " # Check for API key\n", + " api_key = os.getenv('OPENAI_API_KEY')\n", + " if not api_key:\n", + " print(\"⚠️ Note: Set OPENAI_API_KEY environment variable to test LLM extraction\")\n", + " print(\"For this demo, we'll show the configuration only.\\n\")\n", + " \n", + " print(\"📝 Example LLM Configuration with new v0.7.5 features:\")\n", + " print(\"\"\"\n", + "llm_strategy = LLMExtractionStrategy(\n", + " llm_config=LLMConfig(\n", + " provider=\"openai/gpt-4o-mini\",\n", + " api_token=\"your-api-key\",\n", + " temperature=0.7, # 🆕 NEW: Control creativity (0.0-2.0)\n", + " base_url=\"custom-endpoint\" # 🆕 NEW: Custom API endpoint\n", + " ),\n", + " schema=Article.schema(),\n", + " extraction_type=\"schema\",\n", + " instruction=\"Extract article information\"\n", + ")\n", + " \"\"\")\n", + " return\n", + " \n", + " # Create LLM extraction strategy with custom temperature\n", + " llm_strategy = LLMExtractionStrategy(\n", + " llm_config=LLMConfig(\n", + " provider=\"openai/gpt-4o-mini\",\n", + " api_token=api_key,\n", + " temperature=0.3, # 🆕 Lower temperature for more focused extraction\n", + " ),\n", + " schema=Article.schema(),\n", + " extraction_type=\"schema\",\n", + " instruction=\"Extract the article title, a brief summary, and main topics discussed.\"\n", + " )\n", + " \n", + " config = CrawlerRunConfig(\n", + " extraction_strategy=llm_strategy,\n", + " cache_mode=CacheMode.BYPASS\n", + " )\n", + " \n", + " async with AsyncWebCrawler() as crawler:\n", + " result = await crawler.arun(\n", + " url=\"https://en.wikipedia.org/wiki/Artificial_intelligence\",\n", + " config=config\n", + " )\n", + " \n", + " if result.success:\n", + " print(\"\\n✅ LLM Extraction Successful!\")\n", + " print(f\"\\n📄 Extracted Content:\")\n", + " print(result.extracted_content)\n", + " else:\n", + " print(f\"\\n❌ Extraction failed: {result.error_message}\")\n", + " \n", + " print(\"\\n\" + \"=\" * 60)\n", + " print(\"✅ Enhanced LLM Demo Complete!\\n\")\n", + "\n", + "# Run the demo\n", + "await demo_enhanced_llm()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 🔧 Feature 3: Docker Hooks System (NEW! 🆕)\n", + "\n", + "### What is it?\n", + "v0.7.5 introduces a **completely new Docker Hooks System** that lets you inject custom Python functions at 8 key points in the crawling pipeline. This gives you full control over:\n", + "- Authentication setup\n", + "- Performance optimization\n", + "- Content processing\n", + "- Custom behavior at each stage\n", + "\n", + "### Three Ways to Use Docker Hooks\n", + "\n", + "The Docker Hooks System offers three approaches, all part of this new feature:\n", + "\n", + "1. **String-based hooks** - Write hooks as strings for REST API\n", + "2. **Using `hooks_to_string()` utility** - Convert Python functions to strings\n", + "3. **Docker Client auto-conversion** - Pass functions directly (most convenient)\n", + "\n", + "All three approaches are NEW in v0.7.5!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating Reusable Hook Functions\n", + "\n", + "First, let's create some hook functions that we can reuse:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Reusable hook library created!\n", + "\n", + "📚 Available hooks:\n", + " • block_images_hook - Speed optimization\n", + " • set_viewport_hook - Consistent rendering\n", + " • add_custom_headers_hook - Custom headers\n", + " • scroll_page_hook - Lazy content loading\n", + " • log_page_metrics_hook - Page analytics\n" + ] + } + ], + "source": [ + "# Define reusable hooks as Python functions\n", + "\n", + "async def block_images_hook(page, context, **kwargs):\n", + " \"\"\"\n", + " Performance optimization: Block images to speed up crawling\n", + " \"\"\"\n", + " print(\"[Hook] Blocking images for faster loading...\")\n", + " await context.route(\n", + " \"**/*.{png,jpg,jpeg,gif,webp,svg,ico}\",\n", + " lambda route: route.abort()\n", + " )\n", + " return page\n", + "\n", + "async def set_viewport_hook(page, context, **kwargs):\n", + " \"\"\"\n", + " Set consistent viewport size for rendering\n", + " \"\"\"\n", + " print(\"[Hook] Setting viewport to 1920x1080...\")\n", + " await page.set_viewport_size({\"width\": 1920, \"height\": 1080})\n", + " return page\n", + "\n", + "async def add_custom_headers_hook(page, context, url, **kwargs):\n", + " \"\"\"\n", + " Add custom headers before navigation\n", + " \"\"\"\n", + " print(f\"[Hook] Adding custom headers for {url}...\")\n", + " await page.set_extra_http_headers({\n", + " 'X-Crawl4AI-Version': '0.7.5',\n", + " 'X-Custom-Header': 'docker-hooks-demo',\n", + " 'Accept-Language': 'en-US,en;q=0.9'\n", + " })\n", + " return page\n", + "\n", + "async def scroll_page_hook(page, context, **kwargs):\n", + " \"\"\"\n", + " Scroll page to load lazy-loaded content\n", + " \"\"\"\n", + " print(\"[Hook] Scrolling page to load lazy content...\")\n", + " await page.evaluate(\"window.scrollTo(0, document.body.scrollHeight)\")\n", + " await page.wait_for_timeout(1000)\n", + " await page.evaluate(\"window.scrollTo(0, 0)\")\n", + " await page.wait_for_timeout(500)\n", + " return page\n", + "\n", + "async def log_page_metrics_hook(page, context, **kwargs):\n", + " \"\"\"\n", + " Log page metrics before extracting HTML\n", + " \"\"\"\n", + " metrics = await page.evaluate('''\n", + " () => ({\n", + " images: document.images.length,\n", + " links: document.links.length,\n", + " scripts: document.scripts.length,\n", + " title: document.title\n", + " })\n", + " ''')\n", + " print(f\"[Hook] Page Metrics - Title: {metrics['title']}\")\n", + " print(f\" Images: {metrics['images']}, Links: {metrics['links']}, Scripts: {metrics['scripts']}\")\n", + " return page\n", + "\n", + "print(\"✅ Reusable hook library created!\")\n", + "print(\"\\n📚 Available hooks:\")\n", + "print(\" • block_images_hook - Speed optimization\")\n", + "print(\" • set_viewport_hook - Consistent rendering\")\n", + "print(\" • add_custom_headers_hook - Custom headers\")\n", + "print(\" • scroll_page_hook - Lazy content loading\")\n", + "print(\" • log_page_metrics_hook - Page analytics\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Using hooks_to_string() Utility\n", + "\n", + "The new `hooks_to_string()` utility converts Python function objects to strings that can be sent to the Docker API:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Converted 3 hook functions to string format\n", + "\n", + "📝 Example of converted hook (first 200 chars):\n", + "async def block_images_hook(page, context, **kwargs):\n", + " \"\"\"\n", + " Performance optimization: Block images to speed up crawling\n", + " \"\"\"\n", + " print(\"[Hook] Blocking images for faster loading...\")\n", + " awai...\n", + "\n", + "💡 Benefits of hooks_to_string():\n", + " ✓ Write hooks as Python functions (IDE support, type checking)\n", + " ✓ Automatically converts to string format for Docker API\n", + " ✓ Reusable across projects\n", + " ✓ Easy to test and debug\n" + ] + } + ], + "source": [ + "# Convert functions to strings using the NEW utility\n", + "hooks_as_strings = hooks_to_string({\n", + " \"on_page_context_created\": block_images_hook,\n", + " \"before_goto\": add_custom_headers_hook,\n", + " \"before_retrieve_html\": scroll_page_hook,\n", + "})\n", + "\n", + "print(\"✅ Converted 3 hook functions to string format\")\n", + "print(\"\\n📝 Example of converted hook (first 200 chars):\")\n", + "print(hooks_as_strings[\"on_page_context_created\"][:200] + \"...\")\n", + "\n", + "print(\"\\n💡 Benefits of hooks_to_string():\")\n", + "print(\" ✓ Write hooks as Python functions (IDE support, type checking)\")\n", + "print(\" ✓ Automatically converts to string format for Docker API\")\n", + "print(\" ✓ Reusable across projects\")\n", + "print(\" ✓ Easy to test and debug\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 8 Available Hook Points\n", + "\n", + "The Docker Hooks System provides 8 strategic points where you can inject custom behavior:\n", + "\n", + "1. **on_browser_created** - Browser initialization\n", + "2. **on_page_context_created** - Page context setup\n", + "3. **on_user_agent_updated** - User agent configuration\n", + "4. **before_goto** - Pre-navigation setup\n", + "5. **after_goto** - Post-navigation processing\n", + "6. **on_execution_started** - JavaScript execution start\n", + "7. **before_retrieve_html** - Pre-extraction processing\n", + "8. **before_return_html** - Final HTML processing" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Complete Docker Hooks Demo\n", + "\n", + "**Note**: For a complete demonstration of all Docker Hooks approaches including:\n", + "- String-based hooks with REST API\n", + "- hooks_to_string() utility usage\n", + "- Docker Client with automatic conversion\n", + "- Complete pipeline with all 8 hook points\n", + "\n", + "See the separate file: **`v0.7.5_docker_hooks_demo.py`**\n", + "\n", + "This standalone Python script provides comprehensive, runnable examples of the entire Docker Hooks System." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 🛠️ Feature 4: Bug Fixes Summary\n", + "\n", + "### Major Fixes in v0.7.5\n", + "\n", + "1. **URL Processing** - Fixed '+' sign preservation in query parameters\n", + "2. **Proxy Configuration** - Enhanced proxy string parsing (old parameter deprecated)\n", + "3. **Docker Error Handling** - Better error messages with status codes\n", + "4. **Memory Management** - Fixed leaks in long-running sessions\n", + "5. **JWT Authentication** - Fixed Docker JWT validation\n", + "6. **Playwright Stealth** - Fixed stealth features\n", + "7. **API Configuration** - Fixed config handling\n", + "8. **Deep Crawl Strategy** - Resolved JSON encoding errors\n", + "9. **LLM Provider Support** - Fixed custom provider integration\n", + "10. **Performance** - Resolved backoff strategy failures\n", + "\n", + "### New Proxy Configuration Example" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ New proxy configuration format demonstrated\n", + "\n", + "📝 Benefits:\n", + " • More explicit and clear\n", + " • Better authentication support\n", + " • Consistent with industry standards\n" + ] + } + ], + "source": [ + "# OLD WAY (Deprecated)\n", + "# browser_config = BrowserConfig(proxy=\"http://proxy:8080\")\n", + "\n", + "# NEW WAY (v0.7.5)\n", + "browser_config_with_proxy = BrowserConfig(\n", + " proxy_config={\n", + " \"server\": \"http://proxy.example.com:8080\",\n", + " \"username\": \"optional-username\", # Optional\n", + " \"password\": \"optional-password\" # Optional\n", + " }\n", + ")\n", + "\n", + "print(\"✅ New proxy configuration format demonstrated\")\n", + "print(\"\\n📝 Benefits:\")\n", + "print(\" • More explicit and clear\")\n", + "print(\" • Better authentication support\")\n", + "print(\" • Consistent with industry standards\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 🎯 Complete Example: Combining Multiple Features\n", + "\n", + "Let's create a real-world example that uses multiple v0.7.5 features together:" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🎯 Complete v0.7.5 Feature Demo\n", + "\n", + "============================================================\n", + "\n", + "1️⃣ Using Docker Hooks System (NEW!)\n", + " ✓ Converted 3 hooks to string format\n", + " ✓ Ready to send to Docker API\n", + "\n", + "2️⃣ Enabling HTTPS Preservation\n", + " ✓ HTTPS preservation enabled\n", + "\n", + "3️⃣ Using New Proxy Configuration Format\n", + " ✓ New proxy config format ready\n", + "\n", + "4️⃣ Executing Crawl with All Features\n" + ] + }, + { + "data": { + "text/html": [ + "
[INIT].... → Crawl4AI 0.7.5 \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;36m[\u001b[0m\u001b[36mINIT\u001b[0m\u001b[1;36m]\u001b[0m\u001b[36m...\u001b[0m\u001b[36m. → Crawl4AI \u001b[0m\u001b[1;36m0.7\u001b[0m\u001b[36m.\u001b[0m\u001b[1;36m5\u001b[0m\u001b[36m \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[FETCH]... ↓ https://example.com                                                                                  |\n",
+       "✓ | ⏱: 1.29s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mFETCH\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m...\u001b[0m\u001b[32m ↓ \u001b[0m\u001b[4;32mhttps://example.com\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m1.\u001b[0m\u001b[32m29s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[SCRAPE].. ◆ https://example.com                                                                                  |\n",
+       "✓ | ⏱: 0.00s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mSCRAPE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m.. ◆ \u001b[0m\u001b[4;32mhttps://example.com\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m0.\u001b[0m\u001b[32m00s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
[COMPLETE] ● https://example.com                                                                                  |\n",
+       "✓ | ⏱: 1.29s \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;32m[\u001b[0m\u001b[32mCOMPLETE\u001b[0m\u001b[1;32m]\u001b[0m\u001b[32m ● \u001b[0m\u001b[4;32mhttps://example.com\u001b[0m\u001b[32m |\u001b[0m\n", + "\u001b[32m✓\u001b[0m\u001b[32m | ⏱: \u001b[0m\u001b[1;32m1.\u001b[0m\u001b[32m29s \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " ✓ Crawl successful!\n", + "\n", + "📊 Results:\n", + " • Pages crawled: 1\n", + " • Title: Example Domain\n", + " • Content length: 119 characters\n", + " • Links found: 0\n", + "\n", + "============================================================\n", + "✅ Complete Feature Demo Finished!\n", + "\n" + ] + } + ], + "source": [ + "async def complete_demo():\n", + " \"\"\"\n", + " Comprehensive demo combining multiple v0.7.5 features\n", + " \"\"\"\n", + " print(\"🎯 Complete v0.7.5 Feature Demo\\n\")\n", + " print(\"=\" * 60)\n", + " \n", + " # Use function-based hooks (NEW Docker Hooks System)\n", + " print(\"\\n1️⃣ Using Docker Hooks System (NEW!)\")\n", + " hooks = {\n", + " \"on_page_context_created\": set_viewport_hook,\n", + " \"before_goto\": add_custom_headers_hook,\n", + " \"before_retrieve_html\": log_page_metrics_hook\n", + " }\n", + " \n", + " # Convert to strings using the NEW utility\n", + " hooks_strings = hooks_to_string(hooks)\n", + " print(f\" ✓ Converted {len(hooks_strings)} hooks to string format\")\n", + " print(\" ✓ Ready to send to Docker API\")\n", + " \n", + " # Use HTTPS preservation\n", + " print(\"\\n2️⃣ Enabling HTTPS Preservation\")\n", + " url_filter = URLPatternFilter(\n", + " patterns=[r\"^(https:\\/\\/)?example\\.com(\\/.*)?$\"]\n", + " )\n", + " \n", + " config = CrawlerRunConfig(\n", + " exclude_external_links=True,\n", + " preserve_https_for_internal_links=True, # v0.7.5 feature\n", + " cache_mode=CacheMode.BYPASS,\n", + " deep_crawl_strategy=BFSDeepCrawlStrategy(\n", + " max_depth=1,\n", + " max_pages=3,\n", + " filter_chain=FilterChain([url_filter])\n", + " )\n", + " )\n", + " print(\" ✓ HTTPS preservation enabled\")\n", + " \n", + " # Use new proxy config format\n", + " print(\"\\n3️⃣ Using New Proxy Configuration Format\")\n", + " browser_config = BrowserConfig(\n", + " headless=True,\n", + " # proxy_config={ # Uncomment if you have a proxy\n", + " # \"server\": \"http://proxy:8080\"\n", + " # }\n", + " )\n", + " print(\" ✓ New proxy config format ready\")\n", + " \n", + " # Run the crawl\n", + " print(\"\\n4️⃣ Executing Crawl with All Features\")\n", + " async with AsyncWebCrawler(config=browser_config) as crawler:\n", + " # With deep_crawl_strategy, returns a list\n", + " results = await crawler.arun(\n", + " url=\"https://example.com\",\n", + " config=config\n", + " )\n", + " \n", + " if results and len(results) > 0:\n", + " result = results[0] # Get first result\n", + " print(\" ✓ Crawl successful!\")\n", + " print(f\"\\n📊 Results:\")\n", + " print(f\" • Pages crawled: {len(results)}\")\n", + " print(f\" • Title: {result.metadata.get('title', 'N/A')}\")\n", + " print(f\" • Content length: {len(result.markdown.raw_markdown)} characters\")\n", + " print(f\" • Links found: {len(result.links['internal']) + len(result.links['external'])}\")\n", + " else:\n", + " print(f\" ⚠️ No results returned\")\n", + " \n", + " print(\"\\n\" + \"=\" * 60)\n", + " print(\"✅ Complete Feature Demo Finished!\\n\")\n", + "\n", + "# Run complete demo\n", + "await complete_demo()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 🎓 Summary\n", + "\n", + "### What We Covered\n", + "\n", + "✅ **HTTPS Preservation** - Maintain secure protocols throughout crawling \n", + "✅ **Enhanced LLM Integration** - Custom temperature and provider configuration \n", + "✅ **Docker Hooks System (NEW!)** - Complete pipeline customization with 3 approaches \n", + "✅ **hooks_to_string() Utility (NEW!)** - Convert functions for Docker API \n", + "✅ **Bug Fixes** - New proxy config and multiple improvements \n", + "\n", + "### Key Highlight: Docker Hooks System 🌟\n", + "\n", + "The **Docker Hooks System** is completely NEW in v0.7.5. It offers:\n", + "- 8 strategic hook points in the pipeline\n", + "- 3 ways to use hooks (strings, utility, auto-conversion)\n", + "- Full control over crawling behavior\n", + "- Support for authentication, optimization, and custom processing\n", + "\n", + "### Next Steps\n", + "\n", + "1. **Docker Hooks Demo** - See `v0.7.5_docker_hooks_demo.py` for complete Docker Hooks examples\n", + "2. **Documentation** - Visit [docs.crawl4ai.com](https://docs.crawl4ai.com) for full reference\n", + "3. **Examples** - Check [GitHub examples](https://github.com/unclecode/crawl4ai/tree/main/docs/examples)\n", + "4. **Community** - Join [Discord](https://discord.gg/jP8KfhDhyN) for support\n", + "\n", + "---\n", + "\n", + "## 📚 Resources\n", + "\n", + "- 📖 [Full Documentation](https://docs.crawl4ai.com)\n", + "- 🐙 [GitHub Repository](https://github.com/unclecode/crawl4ai)\n", + "- 📝 [Release Notes](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.5.md)\n", + "- 💬 [Discord Community](https://discord.gg/jP8KfhDhyN)\n", + "- 🐦 [Twitter](https://x.com/unclecode)\n", + "\n", + "---\n", + "\n", + "**Happy Crawling with v0.7.5! 🚀**" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 21c302f4390544edfa1a0f61941552b10433d9c0 Mon Sep 17 00:00:00 2001 From: Aravind Karnam Date: Mon, 13 Oct 2025 16:45:16 +0530 Subject: [PATCH 057/119] docs: Add Current sponsors section in README file --- README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/README.md b/README.md index 16fa42a1..476aaee9 100644 --- a/README.md +++ b/README.md @@ -922,3 +922,36 @@ For more details, see our [full mission statement](./MISSION.md). ## Star History [![Star History Chart](https://api.star-history.com/svg?repos=unclecode/crawl4ai&type=Date)](https://star-history.com/#unclecode/crawl4ai&Date) + +--- +## 🌟 Current Sponsors + +### 🏢 Enterprise Sponsors & Partners + +Our enterprise sponsors and technology partners help scale Crawl4AI to power production-grade data pipelines. + +| Company | About | Sponsorship Tier | +|------|------|----------------------------| +| Capsolver | AI-powered Captcha solving service. Supports all major Captcha types, including reCAPTCHA, Cloudflare, and more | 🥈 Silver | +| DataSync | Helps engineers and buyers find, compare, and source electronic & industrial parts in seconds, with specs, pricing, lead times & alternatives.| 🥇 Gold | +| Kidocode | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold | +| Aleph null | Singapore-based Aleph Null is Asia’s leading edtech hub, dedicated to student-centric, AI-driven education—empowering learners with the tools to thrive in a fast-changing world. | 🥇 Gold | + +### 🧑‍🤝 Individual Sponsors + +A heartfelt thanks to our individual supporters! Every contribution helps us keep our opensource mission alive and thriving! + +

+ + + + + + + + +

+ +> Want to join them? [Sponsor Crawl4AI →](https://github.com/sponsors/crawl4ai) + +--- From eea41bf1ca63bc55ce56bcc5a987a56eb869a128 Mon Sep 17 00:00:00 2001 From: Aravind Karnam Date: Mon, 13 Oct 2025 17:00:24 +0530 Subject: [PATCH 058/119] docs: Add a slight background to compensate light theme on github docs --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 476aaee9..913e1d1d 100644 --- a/README.md +++ b/README.md @@ -932,9 +932,9 @@ Our enterprise sponsors and technology partners help scale Crawl4AI to power pro | Company | About | Sponsorship Tier | |------|------|----------------------------| -| Capsolver | AI-powered Captcha solving service. Supports all major Captcha types, including reCAPTCHA, Cloudflare, and more | 🥈 Silver | +| Capsolver | AI-powered Captcha solving service. Supports all major Captcha types, including reCAPTCHA, Cloudflare, and more | 🥈 Silver | | DataSync | Helps engineers and buyers find, compare, and source electronic & industrial parts in seconds, with specs, pricing, lead times & alternatives.| 🥇 Gold | -| Kidocode | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold | +| Kidocode | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold | | Aleph null | Singapore-based Aleph Null is Asia’s leading edtech hub, dedicated to student-centric, AI-driven education—empowering learners with the tools to thrive in a fast-changing world. | 🥇 Gold | ### 🧑‍🤝 Individual Sponsors From 32887ea40d4e4d9d1e989baddaad23a4bdfd15ac Mon Sep 17 00:00:00 2001 From: Aravind Karnam Date: Mon, 13 Oct 2025 17:13:52 +0530 Subject: [PATCH 059/119] docs: Adjust background of sponsor logo to compensate for light themes --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 913e1d1d..b2f5ca01 100644 --- a/README.md +++ b/README.md @@ -932,9 +932,9 @@ Our enterprise sponsors and technology partners help scale Crawl4AI to power pro | Company | About | Sponsorship Tier | |------|------|----------------------------| -| Capsolver | AI-powered Captcha solving service. Supports all major Captcha types, including reCAPTCHA, Cloudflare, and more | 🥈 Silver | +| Capsolver | AI-powered Captcha solving service. Supports all major Captcha types, including reCAPTCHA, Cloudflare, and more | 🥈 Silver | | DataSync | Helps engineers and buyers find, compare, and source electronic & industrial parts in seconds, with specs, pricing, lead times & alternatives.| 🥇 Gold | -| Kidocode | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold | +| Kidocode | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold | | Aleph null | Singapore-based Aleph Null is Asia’s leading edtech hub, dedicated to student-centric, AI-driven education—empowering learners with the tools to thrive in a fast-changing world. | 🥇 Gold | ### 🧑‍🤝 Individual Sponsors From 017144c2dd032594548f06abcb0844e822f9d6b8 Mon Sep 17 00:00:00 2001 From: Aravind Karnam Date: Mon, 13 Oct 2025 17:30:22 +0530 Subject: [PATCH 060/119] docs: Adjust background of sponsor logo to compensate for light themes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b2f5ca01..6764f933 100644 --- a/README.md +++ b/README.md @@ -932,7 +932,7 @@ Our enterprise sponsors and technology partners help scale Crawl4AI to power pro | Company | About | Sponsorship Tier | |------|------|----------------------------| -| Capsolver | AI-powered Captcha solving service. Supports all major Captcha types, including reCAPTCHA, Cloudflare, and more | 🥈 Silver | +| Capsolver | AI-powered Captcha solving service. Supports all major Captcha types, including reCAPTCHA, Cloudflare, and more | 🥈 Silver | | DataSync | Helps engineers and buyers find, compare, and source electronic & industrial parts in seconds, with specs, pricing, lead times & alternatives.| 🥇 Gold | | Kidocode | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold | | Aleph null | Singapore-based Aleph Null is Asia’s leading edtech hub, dedicated to student-centric, AI-driven education—empowering learners with the tools to thrive in a fast-changing world. | 🥇 Gold | From a720a3a9feb2604774249b88098cb16cb2e6ac67 Mon Sep 17 00:00:00 2001 From: Aravind Karnam Date: Mon, 13 Oct 2025 17:32:34 +0530 Subject: [PATCH 061/119] docs: Adjust background of sponsor logo to compensate for light themes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6764f933..055e8fb9 100644 --- a/README.md +++ b/README.md @@ -932,7 +932,7 @@ Our enterprise sponsors and technology partners help scale Crawl4AI to power pro | Company | About | Sponsorship Tier | |------|------|----------------------------| -| Capsolver | AI-powered Captcha solving service. Supports all major Captcha types, including reCAPTCHA, Cloudflare, and more | 🥈 Silver | +| Capsolver | AI-powered Captcha solving service. Supports all major Captcha types, including reCAPTCHA, Cloudflare, and more | 🥈 Silver | | DataSync | Helps engineers and buyers find, compare, and source electronic & industrial parts in seconds, with specs, pricing, lead times & alternatives.| 🥇 Gold | | Kidocode | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold | | Aleph null | Singapore-based Aleph Null is Asia’s leading edtech hub, dedicated to student-centric, AI-driven education—empowering learners with the tools to thrive in a fast-changing world. | 🥇 Gold | From 38a07427086de32ffbed2ed1bc366d55c325fcd0 Mon Sep 17 00:00:00 2001 From: Aravind Karnam Date: Mon, 13 Oct 2025 17:41:19 +0530 Subject: [PATCH 062/119] docs: Adjust background of sponsor logo to compensate for light themes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 055e8fb9..98380a65 100644 --- a/README.md +++ b/README.md @@ -934,7 +934,7 @@ Our enterprise sponsors and technology partners help scale Crawl4AI to power pro |------|------|----------------------------| | Capsolver | AI-powered Captcha solving service. Supports all major Captcha types, including reCAPTCHA, Cloudflare, and more | 🥈 Silver | | DataSync | Helps engineers and buyers find, compare, and source electronic & industrial parts in seconds, with specs, pricing, lead times & alternatives.| 🥇 Gold | -| Kidocode | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold | +| KidoCode | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold | | Aleph null | Singapore-based Aleph Null is Asia’s leading edtech hub, dedicated to student-centric, AI-driven education—empowering learners with the tools to thrive in a fast-changing world. | 🥇 Gold | ### 🧑‍🤝 Individual Sponsors From 6aff0e55aa502ad4f9e9ebb895f8057053033558 Mon Sep 17 00:00:00 2001 From: Aravind Karnam Date: Mon, 13 Oct 2025 17:42:29 +0530 Subject: [PATCH 063/119] docs: Adjust background of sponsor logo to compensate for light themes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 98380a65..72e3070a 100644 --- a/README.md +++ b/README.md @@ -934,7 +934,7 @@ Our enterprise sponsors and technology partners help scale Crawl4AI to power pro |------|------|----------------------------| | Capsolver | AI-powered Captcha solving service. Supports all major Captcha types, including reCAPTCHA, Cloudflare, and more | 🥈 Silver | | DataSync | Helps engineers and buyers find, compare, and source electronic & industrial parts in seconds, with specs, pricing, lead times & alternatives.| 🥇 Gold | -| KidoCode | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold | +| KidoCode | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold | | Aleph null | Singapore-based Aleph Null is Asia’s leading edtech hub, dedicated to student-centric, AI-driven education—empowering learners with the tools to thrive in a fast-changing world. | 🥇 Gold | ### 🧑‍🤝 Individual Sponsors From 8d364a0731e499e4b2de9a0f05aa64338a757738 Mon Sep 17 00:00:00 2001 From: Aravind Karnam Date: Mon, 13 Oct 2025 17:45:10 +0530 Subject: [PATCH 064/119] docs: Adjust background of sponsor logo to compensate for light themes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 72e3070a..02f08a21 100644 --- a/README.md +++ b/README.md @@ -934,7 +934,7 @@ Our enterprise sponsors and technology partners help scale Crawl4AI to power pro |------|------|----------------------------| | Capsolver | AI-powered Captcha solving service. Supports all major Captcha types, including reCAPTCHA, Cloudflare, and more | 🥈 Silver | | DataSync | Helps engineers and buyers find, compare, and source electronic & industrial parts in seconds, with specs, pricing, lead times & alternatives.| 🥇 Gold | -| KidoCode | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold | +| Kidocode

KidoCode

 | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold | | Aleph null | Singapore-based Aleph Null is Asia’s leading edtech hub, dedicated to student-centric, AI-driven education—empowering learners with the tools to thrive in a fast-changing world. | 🥇 Gold | ### 🧑‍🤝 Individual Sponsors From eb257c2ba34575f66cb35fec87a5b5ea3b00a821 Mon Sep 17 00:00:00 2001 From: Aravind Karnam Date: Mon, 13 Oct 2025 17:47:42 +0530 Subject: [PATCH 065/119] docs: fixed sponsorship link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 02f08a21..5e85d562 100644 --- a/README.md +++ b/README.md @@ -952,6 +952,6 @@ A heartfelt thanks to our individual supporters! Every contribution helps us kee

-> Want to join them? [Sponsor Crawl4AI →](https://github.com/sponsors/crawl4ai) +> Want to join them? [Sponsor Crawl4AI →](https://github.com/sponsors/unclecode) --- From c91b235cb730ab9d9b2c05d84f5263fca0d4f594 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Tue, 14 Oct 2025 13:49:57 +0800 Subject: [PATCH 066/119] docs: Update 0.7.5 video walkthrough --- .../v0.7.5_docker_hooks_demo.py | 64 +++++++++---------- .../v0.7.5_video_walkthrough.ipynb | 56 ++++++++-------- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/docs/releases_review/v0.7.5_docker_hooks_demo.py b/docs/releases_review/v0.7.5_docker_hooks_demo.py index 9b4be0c2..6dbe23f9 100644 --- a/docs/releases_review/v0.7.5_docker_hooks_demo.py +++ b/docs/releases_review/v0.7.5_docker_hooks_demo.py @@ -32,8 +32,8 @@ from crawl4ai import hooks_to_string from crawl4ai.docker_client import Crawl4aiDockerClient # Configuration -# DOCKER_URL = "http://localhost:11235" -DOCKER_URL = "http://localhost:11234" +DOCKER_URL = "http://localhost:11235" +# DOCKER_URL = "http://localhost:11234" TEST_URLS = [ # "https://httpbin.org/html", "https://www.kidocode.com", @@ -573,7 +573,7 @@ async def main(): ("String-Based Hooks (REST API)", demo_1_string_based_hooks, False), ("hooks_to_string() Utility", demo_2_hooks_to_string_utility, False), ("Docker Client Auto-Conversion", demo_3_docker_client_auto_conversion, True), - ("Complete Hook Pipeline", demo_4_complete_hook_pipeline, True), + # ("Complete Hook Pipeline", demo_4_complete_hook_pipeline, True), ] for i, (name, demo_func, is_async) in enumerate(demos, 1): @@ -592,7 +592,7 @@ async def main(): # Pause between demos (except the last one) if i < len(demos): print("\n⏸️ Press Enter to continue to next demo...") - input() + # input() except KeyboardInterrupt: print(f"\n⏹️ Demo interrupted by user") @@ -605,40 +605,40 @@ async def main(): continue # Final summary - # print("\n" + "=" * 70) - # print(" 🎉 All Demonstrations Complete!") - # print("=" * 70) + print("\n" + "=" * 70) + print(" 🎉 All Demonstrations Complete!") + print("=" * 70) - # print("\n📊 Summary of v0.7.5 Docker Hooks System:") - # print("\n🆕 COMPLETELY NEW FEATURE in v0.7.5:") - # print(" The Docker Hooks System lets you customize the crawling pipeline") - # print(" with user-provided Python functions at 8 strategic points.") + print("\n📊 Summary of v0.7.5 Docker Hooks System:") + print("\n🆕 COMPLETELY NEW FEATURE in v0.7.5:") + print(" The Docker Hooks System lets you customize the crawling pipeline") + print(" with user-provided Python functions at 8 strategic points.") - # print("\n✨ Three Ways to Use Docker Hooks (All NEW!):") - # print(" 1. String-based - Write hooks as strings for REST API") - # print(" 2. hooks_to_string() - Convert Python functions to strings") - # print(" 3. Docker Client - Automatic conversion (RECOMMENDED)") + print("\n✨ Three Ways to Use Docker Hooks (All NEW!):") + print(" 1. String-based - Write hooks as strings for REST API") + print(" 2. hooks_to_string() - Convert Python functions to strings") + print(" 3. Docker Client - Automatic conversion (RECOMMENDED)") - # print("\n💡 Key Benefits:") - # print(" ✓ Full IDE support (autocomplete, syntax highlighting)") - # print(" ✓ Type checking and linting") - # print(" ✓ Easy to test and debug") - # print(" ✓ Reusable across projects") - # print(" ✓ Complete pipeline control") + print("\n💡 Key Benefits:") + print(" ✓ Full IDE support (autocomplete, syntax highlighting)") + print(" ✓ Type checking and linting") + print(" ✓ Easy to test and debug") + print(" ✓ Reusable across projects") + print(" ✓ Complete pipeline control") - # print("\n🎯 8 Hook Points Available:") - # print(" • on_browser_created, on_page_context_created") - # print(" • on_user_agent_updated, before_goto, after_goto") - # print(" • on_execution_started, before_retrieve_html, before_return_html") + print("\n🎯 8 Hook Points Available:") + print(" • on_browser_created, on_page_context_created") + print(" • on_user_agent_updated, before_goto, after_goto") + print(" • on_execution_started, before_retrieve_html, before_return_html") - # print("\n📚 Resources:") - # print(" • Docs: https://docs.crawl4ai.com") - # print(" • GitHub: https://github.com/unclecode/crawl4ai") - # print(" • Discord: https://discord.gg/jP8KfhDhyN") + print("\n📚 Resources:") + print(" • Docs: https://docs.crawl4ai.com") + print(" • GitHub: https://github.com/unclecode/crawl4ai") + print(" • Discord: https://discord.gg/jP8KfhDhyN") - # print("\n" + "=" * 70) - # print(" Happy Crawling with v0.7.5! 🕷️") - # print("=" * 70 + "\n") + print("\n" + "=" * 70) + print(" Happy Crawling with v0.7.5! 🕷️") + print("=" * 70 + "\n") if __name__ == "__main__": diff --git a/docs/releases_review/v0.7.5_video_walkthrough.ipynb b/docs/releases_review/v0.7.5_video_walkthrough.ipynb index a57de4c9..16738cc7 100644 --- a/docs/releases_review/v0.7.5_video_walkthrough.ipynb +++ b/docs/releases_review/v0.7.5_video_walkthrough.ipynb @@ -62,7 +62,33 @@ "source": [ "---\n", "\n", - "## 🔒 Feature 1: HTTPS Preservation for Internal Links\n", + "## 🔧 Feature 1: Docker Hooks System (NEW! 🆕)\n", + "\n", + "### What is it?\n", + "v0.7.5 introduces a **completely new Docker Hooks System** that lets you inject custom Python functions at 8 key points in the crawling pipeline. This gives you full control over:\n", + "- Authentication setup\n", + "- Performance optimization\n", + "- Content processing\n", + "- Custom behavior at each stage\n", + "\n", + "### Three Ways to Use Docker Hooks\n", + "\n", + "The Docker Hooks System offers three approaches, all part of this new feature:\n", + "\n", + "1. **String-based hooks** - Write hooks as strings for REST API\n", + "2. **Using `hooks_to_string()` utility** - Convert Python functions to strings\n", + "3. **Docker Client auto-conversion** - Pass functions directly (most convenient)\n", + "\n", + "All three approaches are NEW in v0.7.5!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 🔒 Feature 2: HTTPS Preservation for Internal Links\n", "\n", "### Problem\n", "When crawling HTTPS sites, internal links sometimes get downgraded to HTTP, breaking authentication and causing security warnings.\n", @@ -416,7 +442,7 @@ "source": [ "---\n", "\n", - "## 🤖 Feature 2: Enhanced LLM Integration\n", + "## 🤖 Feature 3: Enhanced LLM Integration\n", "\n", "### What's New\n", "- Custom `temperature` parameter for creativity control\n", @@ -979,32 +1005,6 @@ "await demo_enhanced_llm()" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## 🔧 Feature 3: Docker Hooks System (NEW! 🆕)\n", - "\n", - "### What is it?\n", - "v0.7.5 introduces a **completely new Docker Hooks System** that lets you inject custom Python functions at 8 key points in the crawling pipeline. This gives you full control over:\n", - "- Authentication setup\n", - "- Performance optimization\n", - "- Content processing\n", - "- Custom behavior at each stage\n", - "\n", - "### Three Ways to Use Docker Hooks\n", - "\n", - "The Docker Hooks System offers three approaches, all part of this new feature:\n", - "\n", - "1. **String-based hooks** - Write hooks as strings for REST API\n", - "2. **Using `hooks_to_string()` utility** - Convert Python functions to strings\n", - "3. **Docker Client auto-conversion** - Pass functions directly (most convenient)\n", - "\n", - "All three approaches are NEW in v0.7.5!" - ] - }, { "cell_type": "markdown", "metadata": {}, From 9cd06ea7eb3eff0106409454d63009a714f07582 Mon Sep 17 00:00:00 2001 From: Aravind Karnam Date: Fri, 17 Oct 2025 15:30:02 +0530 Subject: [PATCH 067/119] docs: fix order of star history and Current sponsors --- README.md | 9 +- c4ai_menu.json | 1688 +++++++++++++++++++++++++++++++++++++++++++ firecrawl_menu.json | 28 + menu.json | 1688 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 3407 insertions(+), 6 deletions(-) create mode 100644 c4ai_menu.json create mode 100644 firecrawl_menu.json create mode 100644 menu.json diff --git a/README.md b/README.md index 5e85d562..61176532 100644 --- a/README.md +++ b/README.md @@ -919,11 +919,6 @@ We envision a future where AI is powered by real human knowledge, ensuring data For more details, see our [full mission statement](./MISSION.md).
-## Star History - -[![Star History Chart](https://api.star-history.com/svg?repos=unclecode/crawl4ai&type=Date)](https://star-history.com/#unclecode/crawl4ai&Date) - ---- ## 🌟 Current Sponsors ### 🏢 Enterprise Sponsors & Partners @@ -954,4 +949,6 @@ A heartfelt thanks to our individual supporters! Every contribution helps us kee > Want to join them? [Sponsor Crawl4AI →](https://github.com/sponsors/unclecode) ---- +## Star History + +[![Star History Chart](https://api.star-history.com/svg?repos=unclecode/crawl4ai&type=Date)](https://star-history.com/#unclecode/crawl4ai&Date) diff --git a/c4ai_menu.json b/c4ai_menu.json new file mode 100644 index 00000000..810be887 --- /dev/null +++ b/c4ai_menu.json @@ -0,0 +1,1688 @@ +[ + { + "name": "Big Yummy Cheese Burger", + "price": "349", + "description": "A spicy, cheesy indulgence, the Big Yummy Cheese Burger stacks a fiery paneer patty and a rich McCheese patty with crisp lettuce and smoky chipotle sauce on a Quarter Pound bun.. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/916ad567-194a-449a-a9f3-c53acc0fa52e_4dfe7bfa-a200-4eab-9ffe-532236399652.png" + }, + { + "name": "Big Yummy Cheese Meal (M).", + "price": "424.76", + "description": "Double the indulgence, double the flavor: our Big Yummy Cheese Burger meal layers a spicy paneer patty and Cheese patty with crisp lettuce and smoky chipotle sauce, served with fries (M) and a beverage of your choice.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/7c1c4952-781a-4fe7-ad31-d6650fccbbdd_19f5f0fa-2cb3-405c-818a-457c84ba5a01.png" + }, + { + "name": "Big Yummy Chicken Burger", + "price": "349", + "description": "Crafted for true indulgence, tender grilled chicken patty meets the McCrispy chicken patty, elevated with crisp lettuce, jalapenos, and bold chipotle sauce.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/8bdf918d-2571-4f8c-a8f6-bcae8a257144_dbbba789-1e27-4bd0-a356-d7f5ed17c103.png" + }, + { + "name": "Big Yummy Chicken Meal (M).", + "price": "424.76", + "description": "Indulge in double the delight: our Big Yummy Chicken Burger meal pairs the tender grilled chicken patty and Crispy chicken patty with crisp lettuce, jalapeños, and bold chipotle sauce, served with fries (M) and a beverage of your choice ..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/be8e14ab-7861-4cf7-9550-d4a58f09d28c_958f0a00-5a1d-4041-825c-e196ae06d524.png" + }, + { + "name": "Cappuccino (S) + Iced Coffee (S)", + "price": "199.04", + "description": "Get the best coffee combo curated just for you!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/19/5aeea709-728c-43a2-ab00-4e8c754cec74_494372be-b929-496e-8db8-34e128746eb9.png" + }, + { + "name": "Veg Pizza McPuff + McSpicy Chicken Burger", + "price": "260", + "description": "Tender and juicy chicken patty coated in spicy, crispy batter topped with a creamy sauce and crispy shredded lettuce will have you craving for more. Served with Veg Pizza McPuff.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/fb4b9d4775505e82d05d6734ef3e2491" + }, + { + "name": "2 Cappuccino", + "price": "233.33", + "description": "2 Cappuccino (S).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/34aaf6ee-06e2-4c60-9950-8ad569bc5898_2639ffd2-47dd-447b-bbbe-eb391315666f.png" + }, + { + "name": "McChicken Burger + McSpicy Chicken Burger", + "price": "315.23", + "description": "The ultimate chicken combo made just for you. Get the top selling McChicken with the McSpicy Chicken Burger..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/006d51070b0ab9c839a293b87412541c" + }, + { + "name": "2 Iced Coffee", + "price": "233.33", + "description": "Enjoy 2 Iced Coffee.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/e36a9ed2-bfc0-4d36-bfef-297e2c74f991_90e415d8-c4f0-45c1-ad81-db8ef52a5f96.png" + }, + { + "name": "McVeggie Burger + McAloo Tikki Burger", + "price": "210.47", + "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, makes our iconic McVeggie and combo with our top selling McAloo Tikki Burger..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/1f4d583548597d41086df0c723560da7" + }, + { + "name": "Strawberry Shake + Fries (M)", + "price": "196", + "description": "Can't decide what to eat? We've got you covered. Get this snacking combo with Medium Fries and Strawberry Shake..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/74603316fc90ea3cd2b193ab491fbf53" + }, + { + "name": "McChicken Burger + Fries (M)", + "price": "244.76", + "description": "Tender and juicy chicken patty cooked to perfection, with creamy mayonnaise and crunchy lettuce adding flavour to each bite. Served with Medium Fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/a321db80-a223-4a90-9087-054154d27189_9168f1ee-991b-4d8c-8e82-64b2ea249943.png" + }, + { + "name": "McVeggie Burger + Fries (M)", + "price": "215.23", + "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns. Served with Medium Fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/d14cc495747a172686ebe43e675bc941" + }, + { + "name": "McAloo Tikki Burger + Veg Pizza McPuff + Coke", + "price": "190.47", + "description": "The ultimate veg combo made just for you. Get the top selling McAloo Tikki served with Veg Pizza McPuff and Coke..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9f0269a2d28f4918a3b07f63a487f26d" + }, + { + "name": "McAloo Tikki + Fries (R)", + "price": "115.23", + "description": "Aloo Tikki+ Fries (R).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/1ffa9f16-d7ce-48d8-946a-aaac56548c88_10b13615-190f-4f06-91cb-ac7499046fb8.png" + }, + { + "name": "Mexican McAloo Tikki Burger + Fries (R)", + "price": "120", + "description": "A fusion of international taste combined with your favourite aloo tikki patty, layered with shredded onion, and delicious Chipotle sauce. Served with Regular Fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7274d82212a758e597550e8c246fb2f7" + }, + { + "name": "McChicken Burger + Veg Pizza McPuff", + "price": "184.76", + "description": "Tender and juicy chicken patty cooked to perfection, with creamy mayonnaise and crunchy lettuce adding flavour to each bite. Served with Veg Pizza McPuff..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9a66b8ef66d780b9f83a0fc7cd434ded" + }, + { + "name": "McVeggie Burger + Veg Pizza McPuff", + "price": "195.23", + "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns. Served with Veg Pizza McPuff..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/a34eb684-5878-4578-8617-b06e24e46fba_36b736e7-3a91-4618-bf0f-ce60d45e55d2.png" + }, + { + "name": "McVeggie Burger + Fries (R)", + "price": "184.76", + "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns. Served with Regular fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/5e29050f-38f9-4d42-9388-be895f2ba84b_591326c1-a753-44b9-b80c-462196c67fd0.png" + }, + { + "name": "Mexican McAloo Tikki Burger + Fries (L)", + "price": "180", + "description": "A fusion of international taste combined with your favourite aloo tikki patty, layered with shredded onion, and delicious Chipotle sauce. Served with Large Fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f4be6e877d2567a0b585d4b16e53871e" + }, + { + "name": "McChicken Burger + Fries (L)", + "price": "250.47", + "description": "Tender and juicy chicken patty cooked to perfection, with creamy mayonnaise and crunchy lettuce adding flavour to each bite. Served with Large Fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/08e794cb-6520-4907-890c-27449181c9fb_e30fa88b-c5aa-4b71-8c84-135dc433c954.png" + }, + { + "name": "McVeggie Burger + Fries (L)", + "price": "250.47", + "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns. Served with Large fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/b02d421c-fae4-4408-870d-3b51c4e4a94d_0a64d6d8-9eea-452f-917e-988d7db846e2.png" + }, + { + "name": "2 Fries (R)", + "price": "120", + "description": "World Famous Fries, crispy, golden, lightly salted and fried to perfection! Double your happiness with this fries combo.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/4a170da5ecae92e11410a8fbb44c8476" + }, + { + "name": "2 McVeggie Burger", + "price": "270.47", + "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns makes our iconic McVeggie..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/f1edf611-08aa-4b91-b99c-f135eb70df66_ce7bec39-1633-4222-89a2-40013b5d9281.png" + }, + { + "name": "Grilled Chicken & Cheese Burger + Coke", + "price": "239.99", + "description": "Flat 15% Off on Grilled Chicken & Cheese Burger + Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/5/29/18f3ce07-00b5-487b-8608-56440efff007_2b241e0a-fad5-4be4-a848-81c124c95b8b.png" + }, + { + "name": "McAloo Tikki Burger + Veg Pizza McPuff + Fries (R)", + "price": "208.57", + "description": "Flat 15% Off on McAloo Tikki + Veg Pizza McPuff + Fries (R).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/6d7aef29-bae6-403b-a245-38c3371a5363_5dc1ef9c-1f17-4409-946b-b2d78b8710d4.png" + }, + { + "name": "McVeggie Burger + Fries (M) + Piri Piri Mix", + "price": "240", + "description": "Flat 15% Off on McVeggie Burger + Fries (M) + Piri Piri Mix.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/3011bf9d-01fb-4ff3-a2ca-832844aa0dd0_8c231aef-ff56-4dd2-82e5-743a6240c37b.png" + }, + { + "name": "McVeggie Burger + Veg Pizza McPuff + Fries (L)", + "price": "290.47", + "description": "Flat 15% Off on McVeggie Burger + Veg Pizza McPuff + Fries (L).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/5/29/0e4d8ee8-ee6b-4e12-9386-658dbcdc6be4_6810365e-3735-4b21-860b-923a416403be.png" + }, + { + "name": "6 Pc Chicken Nuggets + Fries (M) + Piri Piri Spice Mix", + "price": "247.99", + "description": "The best Non veg sides combo curated for you! Get 6 pc Chicken McNuggets + Fries M. Top it up with Piri Piri mix..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/8cba0938b4401e5c8cb2ccf3741b93c4" + }, + { + "name": "McAloo Tikki Burger + Veg Pizza McPuff + Piri Piri Spice Mix", + "price": "128", + "description": "Get India's favourite burger - McAloo Tikki along with Veg Pizza McPuff and spice it up with a Piri Piri Mix.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/46bcb1e486cbe6dcdb0487b063af58a6" + }, + { + "name": "Grilled Chicken & Cheese Burger + Veg Pizza McPuff", + "price": "196", + "description": "A delicious Grilled Chicken & Cheese Burger + a crispy brown, delicious Pizza McPuff.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/80983b8d-5d94-45f7-84a8-57f2477933de_83c6e531-c828-4ecd-a68a-ed518677fb66.png" + }, + { + "name": "Corn & Cheese Burger + Veg Pizza McPuff", + "price": "184.76", + "description": "A delicious Corn & Cheese Burger + a crispy brown, delicious Pizza McPuff.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/6d92a078-e41e-4126-b70c-b57538675892_c28b61a2-2733-4b3c-a6a2-7943fa109e24.png" + }, + { + "name": "Corn & Cheese Burger + Fries (R)", + "price": "210.47", + "description": "A delicious Corn & Cheese Burger + a side of crispy, golden, world famous fries ??.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/c66d0139-6560-4be9-9239-762f9e80a31a_b4576f89-8186-461d-ba99-28a31a0507f9.png" + }, + { + "name": "Corn & Cheese Burger + Coke", + "price": "239.99", + "description": "Flat 15% Off on Corn & Cheese Burger + Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/5/31/043061fe-2b66-49a4-bc65-b26232584003_ec753203-f35a-487f-bc87-8dcccd31ea3f.png" + }, + { + "name": "Chocolate Flavoured Shake+ Fries (M)", + "price": "196", + "description": "Can't decide what to eat? We've got you covered. Get this snacking combo with Medium Fries and Chocolate Flavoured Shake..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/46781c13d587e5f951ac1bbb39e57154" + }, + { + "name": "2 McFlurry Oreo (S)", + "price": "158", + "description": "Delicious soft serve meets crumbled oreo cookies, a match made in dessert heaven. Make it double with this combo!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/65f3574f53112e9d263dfa924b1f8fed" + }, + { + "name": "2 Hot Fudge Sundae", + "price": "156", + "description": "A sinful delight, soft serve topped with delicious, gooey hot chocolate fudge. So good you won't be able to stop at one!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f1238db9da73d8ec7a999792f35865d9" + }, + { + "name": "McSpicy Chicken Burger + Fries (M) + Piri Piri Spice Mix", + "price": "295.23", + "description": "Tender and juicy chicken patty coated in spicy, crispy batter topped with a creamy sauce and crispy shredded lettuce will have you craving for more. Served with the spicy piri piri mix and medium fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/e10982204687e18ee6541684365039b8" + }, + { + "name": "McSpicy Paneer Burger + Fries (M) + Piri Piri Spice Mix", + "price": "295.23", + "description": "Rich and filling cottage cheese patty coated in spicy, crispy batter topped with a creamy sauce and crispy shredded lettuce will have you craving for more. Served with the spicy piri piri mix and medium fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/87ce9779f986dcb21ab1fcfe794938d1" + }, + { + "name": "McSpicy Paneer + Cheesy Fries", + "price": "295.23", + "description": "Rich and filling cottage cheese patty coated in spicy, crispy batter topped with a creamy sauce and crispy shredded lettuce will have you craving for more. Served with Cheesy Fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/fcc2fb1635f8e14c69b57126014f0bd5" + }, + { + "name": "Black Forest Mcflurry (M) BOGO", + "price": "139", + "description": "Get 2 Black Forest McFlurry for the price of one!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/15/0319a787-bf68-4cca-a81c-f57d9d993918_9b42d2bc-f1bd-47d4-a5a0-6eb9944ec5cd.png" + }, + { + "name": "New McSaver Chicken Surprise", + "price": "119", + "description": "Enjoy a delicious combo of the new Chicken Surprise Burger with a beverage, now in a delivery friendly reusable bottle.. Contains: Sulphite, Soybeans, Milk, Egg, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/b8c0d92e-86b0-4efa-81c1-66be9b3f846b_ddd80150-1aa4-465c-8453-7f7e7650c6c9.png" + }, + { + "name": "New McSaver Chicken Nuggets (4 Pc)", + "price": "119", + "description": "Enjoy New McSaver Chicken Nuggets (4 Pc).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7bf83367ed61708817caefbc79a3c9eb" + }, + { + "name": "New McSaver McAloo Tikki", + "price": "119", + "description": "Enjoy New McSaver McAloo Tikki.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/ab4c47366f0e51ac0071f705b0f2d93e" + }, + { + "name": "New McSaver Pizza McPuff", + "price": "119", + "description": "Enjoy New McSaver Pizza McPuff.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/45fa406e76418771de26c37e8863fbb3" + }, + { + "name": "Chicken Surprise Burger + McChicken Burger", + "price": "204.76", + "description": "Enjoy the newly launched Chicken Surprise Burger with the iconic McChicken Burger.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/4/f1d9557a-6f86-4eb2-abd5-569f2e865de2_9b71c67a-45f1-41c1-9e1c-b80a934768da.png" + }, + { + "name": "Chicken Surprise Burger + Fries (M)", + "price": "170.47", + "description": "Enjoy the newly launched Chicken Surprise Burger with the iconic Fries (M).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/4/ccceb88b-2015-47a3-857b-21a9641d87ed_ad91cfe4-3727-4f23-a34e-4d09c8330709.png" + }, + { + "name": "Crispy Veggie Burger + Cheesy Fries", + "price": "320", + "description": "Feel the crunch with our newly launched Crispy Veggie Burger with Cheesy Fries.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/d81e09c2-e3c9-4b1e-862f-5b760c429a00_fa2d522d-2987-49c7-b924-965ddd970c0e.png" + }, + { + "name": "Crispy Veggie Burger + McAloo Tikki", + "price": "255.23", + "description": "Feel the crunch with our newly launched Crispy Veggie Burger + McAloo Tikki.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/476ed2c4-89b2-41d2-9903-be339a6a07e5_ff0d9d12-ecbd-43b9-97c9-dd3e3c20a5cb.png" + }, + { + "name": "Crispy Veggie Burger + Piri Piri Fries (M)", + "price": "320", + "description": "Feel the crunch with our newly launched Crispy Veggie Burger with Piri Piri Fries (M).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/abda9650-7216-4a8d-87ce-05342438db59_b501b87f-e104-466e-8af6-d3a9947e76ac.png" + }, + { + "name": "Mc Crispy Chicken Burger + Piri Piri Fries (M)", + "price": "360", + "description": "Feel the crunch with our newly launched McCrispy Chicken Burger with Piri Piri Fries (M).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/02bce46b-37fb-4aaa-a035-4daef4fe5350_1292d3ee-7a34-45a2-adc5-1e5b14b4752a.png" + }, + { + "name": "Mc Crispy Chicken Burger + Cheesy Fries", + "price": "370.47", + "description": "Feel the crunch with our newly launched McCrispy Chicken Burger with Cheesy Fries.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/170e8dd8-d9c1-4b67-ab84-cec5dd4a9e32_6d80a13b-024b-454f-b680-d62c7252cd4d.png" + }, + { + "name": "Chicken Surprise Burger + Cold Coffee", + "price": "266.66", + "description": "Start of your morning energetic and satisfied with our new exciting combo of - Chicken Surprise + Cold Coffee (R).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/10/30/01f2c1c4-e80b-48f0-9323-8e75775e08e3_3dd290a6-ed00-405b-a654-e46ed2854c81.png" + }, + { + "name": "McAloo Tikki Burger + Cold Coffee", + "price": "251.42", + "description": "Start of your morning energetic and satisfied with our new exciting combo of - McAloo Tikki +Cold Coffee (R).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/10/30/78de1f5b-7fa2-4e00-b81d-c6d6392cd9ea_6f6eaf41-6844-4349-b37c-49cb7be685b3.png" + }, + { + "name": "Choco Crunch Cookie + McAloo Tikki Burger", + "price": "144.76", + "description": "A crunchy, chocolatey delight meets the iconic Aloo Tikki Burger,sweet and savory, the perfect duo for your snack-time cravings!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/9229f5fb-a89c-49fc-ac40-45e59fdf69dd_e17fe978-1668-4829-812d-d00a1b46e971.png" + }, + { + "name": "Choco Crunch Cookie + McVeggie Burger", + "price": "223.80", + "description": "A crispy Choco Crunch Cookie and a hearty McVeggie Burger,your perfect balance of sweet indulgence and savory delight in every bite!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/61529326-5b1d-4917-b74f-1b3de9d4e8ef_8dbfb34e-5dd7-4a85-b0dd-3f38be677ff9.png" + }, + { + "name": "Lemon Ice Tea + Choco Crunch Cookie", + "price": "237.14", + "description": "A refreshing Lemon Iced Tea paired with a crunchy Choco Crunch Cookie, sweet, zesty, and perfectly balanced for a delightful treat!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/1b46fafc-e69f-4084-8d73-b47f4291e45b_4b0bdb80-9cd2-4b85-a6b5-13fc7348a3a3.png" + }, + { + "name": "Veg Pizza McPuff + Choco Crunch Cookie", + "price": "139.04", + "description": "A perfect snack duo, savoury, Veg Pizza McPuff paired with a crunchy, chocolatey Choco Crunch Cookie for a delicious treat!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/da76bf5a-f86c-44f7-9587-90ef5bdbe973_cf99d6eb-6cef-4f03-bd64-4e9b94523817.png" + }, + { + "name": "Butter Croissant + Cappuccino..", + "price": "209", + "description": "Buttery croissant paired with a rich, frothy cappuccino.Warm, comforting, and perfectly balanced.A timeless duo for your anytime cravings..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/227a6aa3-f18c-4685-bb09-a76a01ec39a1_f347d737-b634-43d7-9b73-dc99bccde65f.png" + }, + { + "name": "Butter Croissant + Iced Coffee.", + "price": "209", + "description": "Buttery, flaky croissant served with smooth, refreshing iced coffee. A classic combo that's light, crisp, and energizing. Perfect for a quick, satisfying bite..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/be3c8688-f975-4d1e-aad7-9229232bcc69_65c9a679-687d-4e66-9847-eb4d5c0f9825.png" + }, + { + "name": "1 Pc Crispy Fried Chicken", + "price": "108", + "description": "Enjoy the incredibly crunchy and juicy and Crispy Fried Chicken- 1 Pc.. Contains: Egg, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/4e7f77ef46d856205d3e4e4913ffc0e9" + }, + { + "name": "2 Crispy Fried Chicken + 2 McSpicy Fried Chicken + 2 Dips + 2 Coke", + "price": "548.99", + "description": "A combo of crunchy, juicy fried chicken and spicy, juicy McSpicy chicken, with 2 Dips and chilled Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/305180a8-d236-4f9f-9d52-852757dfc0f6_ab67db7f-eb5d-4519-88f5-a704b666db4e.png" + }, + { + "name": "2 Pc Crispy Fried Chicken", + "price": "219", + "description": "Enjoy 2 Pcs of the incredibly crunchy and juicy and Crispy Fried Chicken.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/07ed0b2a381f0a21d07888cf1b1216eb" + }, + { + "name": "1 McSpicy Fried Chk + 1 Crispy Fried Chk + 4 Wings + 2 Coke + 2 Dips", + "price": "532.37", + "description": "Juicy and spicy McSpicy chicken, crispy fried chicken, and wings with 2 Dips and Coke perfect for a flavorful meal..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/14c7ed0d-5164-4411-8c52-2151e93801be_4bf5a209-caed-4b44-91b8-48cd71455f2e.png" + }, + { + "name": "1 Pc McSpicy Fried Chicken", + "price": "114", + "description": "Try the new McSpicy Fried chicken that is juicy, crunchy and spicy to the last bite!. Contains: Sulphite, Soybeans, Peanut, Milk, Egg, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/ceb4e0a0-9bff-48f8-a7da-7e8f53c38b86_c9005198-cc85-420a-81ec-7e0bca0ca8ab.png" + }, + { + "name": "2 Pc McSpicy Chicken Wings", + "price": "93", + "description": "Enjoy the 2 pcs of the New McSpicy Chicken Wings. Spicy and crunchy, perfect for your chicken cravings. Contains: Sulphite, Soybeans, Peanut, Milk, Egg, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6e4918a1cf1113361edba3ed33519ffc" + }, + { + "name": "2 Pc McSpicy Fried Chicken", + "price": "217", + "description": "Try the new McSpicy Fried chicken- 2 pcs that is juicy, crunchy and spicy to the last bite!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/057b4bdb-9b88-4c9a-b54a-fc41d0ea1b5f_c6cd0222-69ae-4f30-b588-9741b82d9195.png" + }, + { + "name": "3 Pc McSpicy Fried Chicken Bucket + 1 Coke", + "price": "380.99", + "description": "Share your love for chicken with 3 pcs of McSpicy Fried Chicken with refreshing coke. The perfect meal for your catchup!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/9832febf-e35a-4575-89c1-dfdefd42bb92_42385d00-210d-4554-ac0f-236268e404a3.png" + }, + { + "name": "4 Pc McSpicy Chicken Wings", + "price": "185", + "description": "Enjoy the 4 pcs of the New McSpicy Chicken Wings. Spicy and crunchy, perfect for your chicken cravings.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7122300975cc9640e84cdc7e7c74e042" + }, + { + "name": "4 Pc McSpicy Fried Chicken Bucket", + "price": "455", + "description": "Share your love for chicken with 4 pcs of McSpicy Fried Chicken..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/5cacf92d-f847-4670-acc2-381f984790d1_60f07bc4-f161-4603-b473-527828f8df08.png" + }, + { + "name": "5 Pc McSpicy Fried Chicken Bucket", + "price": "590", + "description": "Share your love for chicken with 5 pcs of McSpicy Fried Chicken that is spicy to the last bite.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/1e617f1f-2573-475f-a50e-8c9dd02dd451_6ccb66a3-728b-4abb-8a03-4f3eed32da18.png" + }, + { + "name": "12 Pc Feast Chicken Bucket", + "price": "808.56", + "description": "Enjoy 12 pc bucket of 4 Pc McSpicy Fried Chicken + 4 Pc Crispy Fried Chicken+ 4 pc McSpicy Chicken Wings + 2 Medium Cokes + 2 Dips. (Serves 3-4).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/a09f66d7-3ad3-4f8c-9168-6a95849430f5_e360b6f1-659f-42c0-abc0-7f05b95496ce.png" + }, + { + "name": "Chicken Lover's Bucket", + "price": "599.04", + "description": "Enjoy this crunchy combination of 4 Pc McSpicy Chicken Wings + 2 Pc McSpicy Fried Chicken + 2 Pc Crispy Fried Chicken+ 2 Dips + 2 Cokes. A chicken lover's dream come true! (Serves 3-4).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/650f5998-10b0-4c07-bb3b-e68036242f11_c9b5e73c-4bbf-4c04-b918-dd7a32b7973b.png" + }, + { + "name": "4 Chicken Wings + 2 McSpicy Fried Chicken + 2 Coke + 2 Dips", + "price": "528.56", + "description": "Spicy, juicy McSpicy Chicken wings and 2 Pc McSpicy Fried chicken with 2 Dips, paired with 2 chilled cokes.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/42ff8854-3701-4b8c-aace-da723977c2e3_8b62f3cc-32eb-4e84-949e-375045125efc.png" + }, + { + "name": "4 McSpicy Fried Chicken Bucket + 2 Dips + 2 Coke", + "price": "532.37", + "description": "4 pieces of juicy, spicy McSpicy Fried Chicken with 2 Dips and the ultimate refreshment of chilled coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/305180a8-d236-4f9f-9d52-852757dfc0f6_ab67db7f-eb5d-4519-88f5-a704b666db4e.png" + }, + { + "name": "5 McSpicy Fried Chicken Bucket + 2 Dips + 2 Coke", + "price": "566.66", + "description": "5 pieces of juicy, spicy McSpicy Fried chicken with 2 Dips and 2 refreshing Cokes..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/29742264-8fee-4b8f-a8e2-52efb5d4edf9_314d6cbc-b4ea-4d92-92ea-5b7d1df70a9a.png" + }, + { + "name": "8 McSpicy Chicken Wings Bucket + 2 Coke + 1 Dip", + "price": "465.71", + "description": "Juicy, spicy McSpicy Chicken wings with 1 Dip and the ultimate refreshment of chilled coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/ea6a4090-5c1d-4b32-926b-5e1de33574ba_242accc3-d883-4434-9913-fb6c35574c9b.png" + }, + { + "name": "Chicken Surprise Burger with Multi-Millet Bun", + "price": "84.15", + "description": "Try the Chicken Surprise Burger in the new multi-millet bun! Enjoy the same tasty chicken patty you love, now sandwiched between a nutritious multi-millet bun.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/1fb900f6-71bb-4724-a507-830735f555c5_06c06358-bc36-469d-84e3-44e9504bb7d0.png" + }, + { + "name": "McAloo Tikki Burger with Multi-Millet Bun", + "price": "83.91", + "description": "Try your favourite McAloo Tikki Burger in a multi-millet bun! Enjoy the same tasty McAloo Tikki patty you love, now sandwiched between a nutritious millet bun..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/cfd7d779-b04d-43be-947e-a43df33ae119_a0de6b13-c9ac-489a-8416-e9b72ace4542.png" + }, + { + "name": "McChicken Burger with Multi-Millet Bun", + "price": "150.47", + "description": "Make a healthier choice with our McChicken Burger in a multi-millet bun! Same juicy chicken patty, now with a nutritious twist..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/bf0a9899-19a5-4fc9-9898-9e15098bac43_681fbed4-5354-4d5a-9d67-b59d274ea633.png" + }, + { + "name": "McSpicy Chicken Burger with Multi-Millet Bun", + "price": "221.76", + "description": "Feel the heat and feel good too! Try your McSpicy Chicken Burger in nutritious multi-millet bun..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/6eac1f61-cf7b-422a-bc6d-8c55bc1ff842_01e557a3-f5a2-4239-b06b-01ffd46ec154.png" + }, + { + "name": "McSpicy Paneer Burger with Multi-Millet Bun", + "price": "221.76", + "description": "Spice up your meal with a healthier bite! Try your McSpicy Paneer Burger with the nutritious multi-millet bun..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/70eb5956-3666-47f4-bcf8-1f5075207222_fd9e2a06-3f27-47e3-88aa-86895ffaa65e.png" + }, + { + "name": "McVeggie Burger with Multi-Millet Bun", + "price": "158.40", + "description": "Try your favorite McVeggie Burger in a nutritious multi-millet bun! A healthier twist on a classic favorite, with the same tasty veggie patty you love.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/397caee8-4da3-4143-98b1-e4dac155ab3e_0ed975df-7665-4f3a-ae16-8a7eb008afd6.png" + }, + { + "name": "Crispy Veggie Burger Protein Plus (1 Slice)", + "price": "226.71", + "description": "A flavourful patty made with a blend of 7 Premium veggies topped with zesty cocktail sauce now a protein slice to fuel you up, all served between soft premium buns. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/2fcff334-211c-4a2b-bee8-018ce9f10572_71ad5b2a-982d-407a-a833-8f3c1c6bf240.png" + }, + { + "name": "Crispy Veggie Burger Protein Plus (2 Slices)", + "price": "253.43", + "description": "A flavourful patty made with a blend of 7 Premium veggies topped with zesty cocktail sauce now a protein slice to fuel you up, all served between soft premium buns. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/e2c0ecf8-1ca0-447c-a56f-1b8758f7a406_314e4442-4db5-403e-b71c-ffa797622eee.png" + }, + { + "name": "Crispy Veggie Burger Protein Plus + Corn + Coke Zero", + "price": "399", + "description": "A flavourful patty made with a blend of 7 premium veggies, topped with zesty cocktail sauce and now a protein slice to fuel you up, all served between soft premium buns. Paired with corn and Coke Zero..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/d0ac58e8-e601-49e1-abf2-a5456a63f057_a8bb7584-906a-4ce1-891b-6e877f6b4543.png" + }, + { + "name": "McEgg Burger Protein Plus (1 Slice)", + "price": "89.10", + "description": "A steamed egg , spicy habanero sauce and onions and a tasty new protein slice. Simple, satisfying and powered with protein.. Contains: Gluten, Milk, Egg, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/0482392e-9b78-466d-93db-e653c77e0e35_55c40d44-9050-47eb-902e-5639c1423ee7.png" + }, + { + "name": "McEgg Burger Protein Plus (2 Slices)", + "price": "117.80", + "description": "A steamed egg , spicy habanero sauce and onions and a tasty new protein slice. Simple, satisfying and powered with protein.. Contains: Gluten, Milk, Egg, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/a5750dc8-8a1f-48a0-8fa2-83007ee4377c_c8501c16-90f4-4943-80f7-719f519fb918.png" + }, + { + "name": "McAloo Tikki Burger Protein Plus (1 Slice)", + "price": "89.10", + "description": "The OG Burger just got an upgrade with a tasty protein slice.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/6c4f7375-3424-4d01-8425-509381fb3015_25ebe6f6-2f23-4b3f-8dcb-02ea740f91e9.png" + }, + { + "name": "McAloo Tikki Burger Protein Plus (2 Slices)", + "price": "117.80", + "description": "The OG Burger just got an upgrade with a tasty protein slice.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/a95ad198-b6d8-43b0-9960-a1b8d8ceb6f2_12b5df65-ba9d-41ff-b762-88e391b121f8.png" + }, + { + "name": "McAloo Tikki Burger Protein Plus+ Corn + Coke Zero", + "price": "299", + "description": "The OG McAloo Tikki Burger just got an upgrade with a tasty protein slice. Served with buttery corn and a refreshing Coke Zero for a nostalgic yet balanced combo..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/da8beade-c083-4396-b4ff-289b647af81d_b0f6b946-7151-45e3-9fe0-e15a2c46c1d6.png" + }, + { + "name": "McCheese Chicken Burger Protein Plus (1 Slice)", + "price": "297", + "description": "Double the indulgence with sinfully oozing cheesey patty and flame grilled chicken patty , along with chipotle sauce , shredded onion , jalapenos , lettuce and now with a protein slice. Indulgent meets protein power.. Contains: Gluten, Egg, Milk, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/d32ea9af-d134-4ae9-8a1a-8dafddd53cb7_da16f662-ce98-4990-ab8c-566697743730.png" + }, + { + "name": "McCheese Chicken Burger Protein Plus (2 Slices)", + "price": "324.72", + "description": "Double the indulgence with sinfully oozing cheesey patty and flame grilled chicken patty , along ith chipotle sauce , shredded onion , jalapenos , lettuce and now with a protein slice. Indulgent meets protein power.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/e2b28b79-63df-43bb-8c41-240a076e0a3a_49dbc6df-3f9d-448c-bd94-377ebcc3a335.png" + }, + { + "name": "McCheese Chicken Burger Protein Plus + 4 Pc Chicken Nuggets+ Coke Zero", + "price": "449", + "description": "Double the indulgence with sinfully oozing cheesy patty and flame-grilled chicken patty, chipotle sauce, shredded onion, jalapenos, lettuce, and now a protein slice. Served with 4 Pc Chicken nuggets and Coke Zero. Indulgent meets protein power..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/f9aae1ec-6313-4684-aeb3-50de503fcf23_f5b200ed-60fa-42f6-928f-6b95327da3bb.png" + }, + { + "name": "McChicken Burger Protein Plus (1 Slice)", + "price": "185.13", + "description": "The classic McChicken you love, made more wholesome with a protein slice. Soft, savoury, and now protein rich.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/20fd3d59-0003-4e80-ad71-79ca8ae4d050_8bf1219b-a67d-4518-a115-167fa10bc440.png" + }, + { + "name": "McChicken Burger Protein Plus + 4 Pc Chicken Nuggets + Coke Zero", + "price": "349", + "description": "The classic McChicken you love, made more wholesome with a protein slice. Soft, savoury, and now protein-rich. Comes with 4 crispy Chicken nuggets and a chilled Coke Zero..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/afdcdd59-c5f0-44f1-9014-b28455ce0ecf_059ea761-4f24-4e04-b152-cdb82a04b366.png" + }, + { + "name": "McChicken Protein Burger Plus (2 Slices)", + "price": "212.84", + "description": "The classic McChicken you love, made more wholesome with a protein slice.Soft, savoury, and now protein-rich.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/1e1f281e-8378-4993-a7c1-d6f319a7bd17_768a7827-1fe8-43a6-ba35-a0ccb5a95ef5.png" + }, + { + "name": "McCrispy Chicken Burger Protein Plus (1 Slice)", + "price": "246.51", + "description": "A Crunchy , golden chicken thigh fillet , topped with fresh lettuce and creamy pepper mayo now also with a hearty protein slice all nestled between soft toasted premium buns.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/4f908fa6-2fa5-4367-9da9-9039cb5e72e0_c8570551-57ae-441a-81c9-64335392ac3b.png" + }, + { + "name": "McCrispy Chicken Burger Protein Plus (2 Slices)", + "price": "276.20", + "description": "A Crunchy , golden chicken thigh fillet , topped with fresh lettuce and creamy pepper mayo now also with a hearty protein slice all nestled between soft toasted premium buns.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/fc4e713d-5ee0-421e-9690-7c8b2c587e5a_2b710d54-3ae7-46fe-990c-140b34494dee.png" + }, + { + "name": "McCrispy Chicken Burger Protein Plus + 4 Pc Chicken Nugget + Coke Zero", + "price": "419", + "description": "A crunchy, golden chicken thigh fillet topped with fresh lettuce and creamy pepper mayo, now also with a hearty protein slice, all nestled between soft toasted premium buns. Comes with 4-piece nuggets and Coke Zero..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/fdb3d9b1-2262-4b18-9264-9bd00b3213ba_20858442-9b85-4728-b7b2-07d72349782e.png" + }, + { + "name": "McEgg Burger Protein Plus + 4 Pc Chicken Nuggets + Coke Zero", + "price": "299", + "description": "A steamed egg, spicy habanero sauce, onions, and a tasty new protein slice. Simple, satisfying, and powered with protein. Served with 4-piece chicken nuggets and Coke Zero..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/47092b74-fac3-48a2-9abc-1d43b975199f_d07e626a-6cb9-4664-8517-4ce7ba71b234.png" + }, + { + "name": "McSpicy Chicken Burger Protein Plus (1 Slice)", + "price": "225.72", + "description": "Indulge in our signature tender chicken patty coated in spicy crispy batter , topped with creamy sauce ,crispy lettuce and now with a new protein slice .. Contains: Gluten, Milk, Egg, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/79035335-855d-4b89-93e4-c063258aaf64_22e9ada8-6768-481d-83d3-7223bc119bce.png" + }, + { + "name": "McSpicy Chicken Burger Protein Plus (2 Slices)", + "price": "252.44", + "description": "Indulge in our signature tender chicken patty coated in spicy crispy batter , topped with creamy sauce ,crispy lettuce and now with a new protein slice .. Contains: Gluten, Egg, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/36e2e35c-86d5-4b0d-b573-325c7b4e5fd9_4bc54386-d43e-4d28-baca-d0da22bc3668.png" + }, + { + "name": "McSpicy Chicken Burger Protein Plus + 4 Pc Chicken Nuggets + Coke Zero", + "price": "399", + "description": "Indulge in our signature tender chicken patty coated in spicy crispy batter , topped with creamy sauce ,crispy lettuce and now with a new protein slice Served with 4-piece chicken nuggets and Coke Zero..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/a81bd599-05ad-4453-9ada-499d9cf45b29_1096d08c-d519-4133-babb-2f54a9b6e5f0.png" + }, + { + "name": "McSpicy Paneer Burger Protein Plus (1 Slice)", + "price": "226.71", + "description": "Indulge in rich and filling spicy paneer patty served with creamy sauce, crispy lettuce and now with a new protein slice. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/3ca8296e-9885-4df6-8621-5349298a150f_a2ad6b56-b6ae-4e4b-986c-abe75e052d75.png" + }, + { + "name": "McSpicy Paneer Burger Protein Plus (2 Slices)", + "price": "253.43", + "description": "Indulge in rich and filling spicy paneer patty served with creamy sauce, crispy lettuce and now with a new protein slice. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/c2561720-1064-4043-a143-3c9ae893e481_b09ff597-1b69-4098-ba04-6ab3f3b09c6f.png" + }, + { + "name": "McSpicy Paneer Burger Protein Plus + Corn + Coke Zero", + "price": "399", + "description": "Indulge in rich and filling spicy paneer patty served with creamy sauce, crispy lettuce and now with a new protein slice. Served with Corn and Coke Zero..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/d5c3f802-ad34-4086-ae66-731406a82c99_4b87555b-5e27-40da-b7f1-fd557212e344.png" + }, + { + "name": "McSpicy Premium Burger Veg Protein Plus (2 Slices)", + "price": "303.93", + "description": "A wholesome spicy paneer patty, lettuce topped with jalapenos and cheese slice and now with a protein-packed slice for that extra boost , spicy cocktail sauce and cheese sauce.. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/05949fc3-08a4-4e47-8fd1-16abc12a952c_0b80fed2-213d-4945-9ea7-38c818cdb6ef.png" + }, + { + "name": "McSpicy Premium Chicken Burger Protein Plus (1 Slice)", + "price": "287.10", + "description": "A wholesome spicy chicken patty lettuce topped with jalapenos and cheese slice plus an added protein slice , spicy cocktail sauce and cheese sauce.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/690ecde3-4611-4323-8436-a1dadfe2eb7b_fe9b5993-f422-4bae-8211-9715dd1d51d8.png" + }, + { + "name": "McSpicy Premium Chicken Burger Protein Plus (2 Slices)", + "price": "315.80", + "description": "A wholesome spicy chicken patty lettuce topped with jalapenos and cheese slice plus an added protein slice , spicy cocktail sauce and cheese sauce.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/80bf28f3-b317-4846-b678-95cfc857331a_d247479e-0bcb-4f5d-90da-b214d30e70a3.png" + }, + { + "name": "McSpicy Premium Chicken Protein Plus + 4 Pc Chicken Nugget + Coke Zero", + "price": "479", + "description": "A wholesome spicy chicken patty, lettuce topped with jalapenos and cheese slice, plus an added protein slice. Comes with spicy cocktail sauce and cheese sauce,served with 4 Pc crispy chicken nuggets and a Coke Zero..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/a512848a-d34d-44cf-a0db-f7e84781c63d_6d8fa5df-a1c6-42e6-b88b-929c59c4f50c.png" + }, + { + "name": "McSpicy Premium Veg Burger Protein Plus (1 Slice)", + "price": "276.20", + "description": "A wholesome spicy paneer patty, lettuce topped with jalapenos and cheese slice and now with a protein packed slice for that extra boost , spicy cocktail sauce and cheese sauce.. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/8768ff6a-ea10-4260-91e6-018695820d79_2399dc04-b793-4336-b2e4-a6240cdc5f78.png" + }, + { + "name": "McSpicy Premium Veg Burger Protein Plus + Corn + Coke Zero", + "price": "449", + "description": "A wholesome spicy paneer patty, lettuce topped with jalapenos and cheese slice, now with a protein-packed slice for that extra boost. Comes with spicy cocktail sauce, cheese sauce, buttery corn, and a chilled Coke Zero..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/d183fbeb-62dc-4b3d-936f-99ddc9dc5466_ce66dc12-da9f-479d-bd4a-c77fdb3dae2c.png" + }, + { + "name": "McVeggie Burger Protein Plus (1 Slice)", + "price": "185.13", + "description": "The classic McVeggie you love, made more wholesome with a protein slice.Soft, savoury, and now protein-rich.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/99aedaa8-836b-4893-b7f9-24b3303b6c94_781b8449-5bc9-42e5-9747-00bcec474deb.png" + }, + { + "name": "McVeggie Burger Protein Plus (2 Slices)", + "price": "212.88", + "description": "The classic McVeggie you love, made more wholesome with a protein slice.Soft, savoury, and now protein rich.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/630f74dd-841a-4641-bf7f-c23cd9e4e0d5_0fd861eb-43f6-46c1-9bfe-f8043c2aae6a.png" + }, + { + "name": "McVeggie Burger Protein Plus + Corn + Coke Zero", + "price": "339", + "description": "The classic McVeggie you love, made more wholesome with a protein slice. Soft, savoury, and now protein rich. Served with sweet corn and Coke Zero for a balanced meal..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/f448d30d-9bbe-4659-bbb4-14b22d93b0ae_0a188512-5619-4d62-980f-7db8e45f6ebe.png" + }, + { + "name": "Big Group Party Combo 6 Veg", + "price": "760.95", + "description": "Enjoy a Big Group Party Combo of McAloo + McVeggie + McSpicy Paneer + Mexican McAloo + Corn and Cheese + Crispy Veggie burger.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/427a977d-1e14-4fe9-a7ac-09a71f578514_e3089568-134d-45fe-b97e-59d756892f15.png" + }, + { + "name": "Big Group Party Combo for 6 Non- Veg", + "price": "856.19", + "description": "Enjoy a Big Group Party Combo of Surprise Chicken + McChicken + McSpicy Chicken + Grilled Chicken + McSpicy Premium + Mc Crispy Chicken Burger .", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/e1a453be-6ae4-4e82-8957-c43756d2d72a_d7cac204-1cb6-4b6d-addf-c0324435cc58.png" + }, + { + "name": "Big Group Party Combo1 for 4 Non- Veg", + "price": "475.23", + "description": "Save on your favourite Big Group Party Combo - Surprise Chicken + McChicken + McSpicy Chicken + Grilled Chicken Burger.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/d955dfa6-3a2c-46d2-b59c-e9e59b0939c3_1417a17c-bb7e-4e24-8931-3950cac2f074.png" + }, + { + "name": "Big Group Party Combo1 for 4 Veg", + "price": "475.23", + "description": "Get the best value in your Combo for 4 Save big on your favourite Big Group Party Combo-McAloo + McVeggie + McSpicy Paneer + Corn and Cheese Burger.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/5c723cbf-e7e2-48cc-a8fe-6f4e0eeea73d_899a4709-e9fb-4d9f-a997-973027fc0e7d.png" + }, + { + "name": "Big Group Party Combo2 for 4 Non-Veg", + "price": "522.85", + "description": "Your favorite party combo of 2 McChicken + 2 McSpicy Chicken Burger.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/b648ba7b-e82d-4d37-ab47-736fdb12cd90_19ef381a-3cc6-4b50-9e69-911f3656987f.png" + }, + { + "name": "2 Crispy Veggie Burger + Fries (L) + 2 Coke", + "price": "635.23", + "description": "Feel the crunch with Burger Combos for 2: 2 Crispy Veggie Burger + Fries (L)+ 2 Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/ce6551d8-829a-4dde-9131-cbf7818a6f26_a5b69cee-6377-4295-bbee-49725693c45d.png" + }, + { + "name": "Big Group Party Combo2 for 4 Veg", + "price": "522.85", + "description": "Your favorite party combo of 2 McVeggie + 2 McSpicy Paneer Burger.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/245c7235-500d-4212-84bd-f60f9aaf27be_73b24a53-d304-4db0-9336-061760ca17a9.png" + }, + { + "name": "2 Crispy Veggie Burger + 2 Fries (M) + Veg Pizza McPuff", + "price": "604.76", + "description": "Feel the crunch with Burger Combos for 2: 2 Crispy Veggie Burger + 2 Fries (M)+ Veg Pizza McPuff.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/ca2023e7-41e0-4233-a92c-c2bc52ef8ee5_33a0ea24-50ee-4a9b-b917-bbc2e6b5f37b.png" + }, + { + "name": "Crispy Veggie Burger + McVeggie Burger + Fries (M)", + "price": "424.76", + "description": "Feel the crunch with Crispy Veggie Burger+ McVeggie + Fries (M).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/c6dd67c1-0af9-4333-a4b5-3501e0ce4579_2fee69c9-f349-46c5-bdd9-8965b9f76687.png" + }, + { + "name": "Burger Combo for 2: McAloo Tikki", + "price": "364.76", + "description": "Stay home, stay safe and share a combo- 2 McAloo Tikki Burgers + 2 Fries (L).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/ea7ba594c7d77cb752de9a730fbcb3bf" + }, + { + "name": "6 Pc Chicken Nuggets + McChicken Burger + Coke", + "price": "375.22", + "description": "Tender and juicy chicken patty cooked to perfection, with creamy mayonnaise and crunchy lettuce adding flavour to each bite. Served with 6 Pc Nuggets and Coke..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/934194567f9c231dc46dccf2d4e6d415" + }, + { + "name": "Burger Combo for 2: McSpicy Chicken + McChicken", + "price": "464.76", + "description": "Flat 15% Off on McSpicy Chicken Burger + McChicken Burger + Fries (M).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/10ada13e-5724-487f-8ab6-fd07005859ad_a57d565d-0dfe-4424-bc5b-77b16143ad63.png" + }, + { + "name": "Burger Combo for 2: Corn & Cheese + McVeggie", + "price": "404.76", + "description": "Flat 15% Off on Corn & Cheese Burger +McVeggie Burger+Fries (M).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/08e9bc73-6774-41cf-96bb-ca817c4e23d3_e00ec89e-86f8-4531-956a-646082dc294c.png" + }, + { + "name": "Burger Combo for 2: McSpicy Chicken Burger with Pizza McPuff", + "price": "535.23", + "description": "Save big on your favourite sharing combo- 2 McSpicy Chicken Burger + 2 Fries (M) + Veg Pizza McPuff.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/e62aea3ba1cd5585a76004f59cd991e5" + }, + { + "name": "Burger Combo for 2: McSpicy Paneer + McAloo Tikki with Pizza McPuff", + "price": "427.61", + "description": "Get the best value in your meal for 2. Save big on your favourite sharing meal - McSpicy Paneer Burger + 2 Fries (M) + McAloo Tikki Burger + Veg Pizza McPuff.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/0ea8a2fddbbc17bc6239a9104963a3e8" + }, + { + "name": "Burger Combo for 2: McChicken Burger", + "price": "464.75", + "description": "Save big on your favourite sharing combo - 2 McChicken Burger + Fries (L) + 2 Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/7/177036e5-4afe-4076-b9cd-d7031f60ffe8_97687ef4-e242-4625-9b3c-398c60b8ddf2.png" + }, + { + "name": "Burger Combo for 2: McSpicy Chicken Burger", + "price": "548.56", + "description": "Save big on your favourite sharing combo - 2 McSpicy Chicken Burger + Fries (L) + 2 Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/98afaf26d81b15bec74cc356fe60cc13" + }, + { + "name": "Burger Combo for 2: McVeggie Burger", + "price": "424.75", + "description": "Save big on your favourite sharing combo - 2 McVeggie Burger + Fries (L) + 2 Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/09b3cb6130cfae15d486223c313fb6c6" + }, + { + "name": "2 Chicken Maharaja Mac Burger + 2 Coke + Fries (L) + McFlurry Oreo (M)", + "price": "670.47", + "description": "Enjoy 2 of the tallest burgers innovated by us. Created with chunky juicy grilled chicken patty paired along with fresh ingredients like jalapeno, onion, slice of cheese, tomatoes & crunchy lettuce dressed with the classical Habanero sauce. Served with Coke, Large Fries and a medium McFlurry Oreo.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/65c9c9b82c4d1f77a05dc4d89c9ead1d" + }, + { + "name": "Burger Combo for 2: Corn & Cheese Burger", + "price": "464.75", + "description": "Save big on your favourite sharing combo - 2 Corn and Cheese Burger + Fries (L) + 2 Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/847b562672e71c2352d92b797c0b0a4e" + }, + { + "name": "Burger Combo for 2: Grilled Chicken & Cheese", + "price": "495.23", + "description": "Save big on your favourite sharing combo - 2 Grilled Chicken and Cheese Burger + Fries (L) + 2 Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/1166a8baa3066342affb829ef0c428dd" + }, + { + "name": "2 Mc Crispy Chicken Burger + Fries (L) + 2 Coke", + "price": "724.75", + "description": "Feel the crunch with our Burger Combos for 2 : 2 McCrispy Chicken Burger + Fries (L)+ 2 Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/b4fda3b0-82c9-4d97-ab9c-c804b7d7893a_a14992d5-49cb-4035-827d-a43e40752840.png" + }, + { + "name": "Mc Crispy Chicken Burger + McChicken Burger + Fries (M)", + "price": "430.47", + "description": "Feel the crunch with McCrispy Chicken Burger+ McChicken + Fries (M).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/eaeeaf30-7bb6-4bef-9b78-643533a4520c_e54bb308-0447-44c2-9aaf-f36ce25f7939.png" + }, + { + "name": "Mc Crispy Chicken Burger + McSpicy Chicken Wings - 2 pc + Coke (M)", + "price": "415.23", + "description": "Feel the crunch with McCrispy Chicken Burger+ McSpicy Chicken Wings - 2 pc + Coke (M).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/f8669731-b3e5-434d-8b45-269b75555b9b_3dfd8e01-5472-40db-a266-bb055036531b.png" + }, + { + "name": "McChicken Double Patty Burger Combo", + "price": "352.99", + "description": "Your favorite McChicken Burger double pattu burger + Fries (M) + Drink of your choice in a new, delivery friendly, resuable bottle..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/e8a8f43ee29a3b97d2fac37e89648eac" + }, + { + "name": "McSpicy Chicken Double Patty Burger combo", + "price": "418.99", + "description": "Your favorite McSpicy Chicken double patty Burger + Fries (M) + Drink of your choice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/91ed96b67df6e630a6830fc2e857b5b1" + }, + { + "name": "McVeggie Burger Happy Meal", + "price": "298.42", + "description": "Enjoy a combo of McVeggie Burger + Sweet Corn + B Natural Mixed Fruit Beverage + Book.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/15e7fc6b-f645-4ed9-a391-1e1edc452f9f_47ad6460-6af4-43b4-8365-35bb0b3fc078.png" + }, + { + "name": "McChicken Burger Happy Meal", + "price": "321.42", + "description": "Enjoy a combo of McChicken Burger + Sweet Corn + B Natural Mixed Fruit Beverage + Book.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/363ca5bf-bb04-4847-bc4e-5330a27d3874_8b5e46ed-61ba-4e8f-b291-2955af07eba0.png" + }, + { + "name": "McAloo Tikki Burger Happy Meal", + "price": "205.42", + "description": "Enjoy a combo of McAloo Tikki Burger + Sweet Corn + B Natural Mixed Fruit Beverage + Book.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/a9f2acb6-3fd3-4691-beb0-61439337f907_6733a4ee-3eb5-46da-9ab7-1b6943f68be7.png" + }, + { + "name": "Big Spicy Paneer Wrap Combo", + "price": "360.99", + "description": "Your favorite Big Spicy Paneer Wrap + Fries (M) + Drink of your choice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/22/116148a2-7f56-44af-bdd0-6bbb46f48baa_355bbd20-86d4-49b8-b3a9-d290772a9676.png" + }, + { + "name": "9 Pc Chicken Nuggets Combo", + "price": "388.98", + "description": "Enjoy your favorite Chicken McNuggets + Fries (M) + Drink of your choice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/4c56f086e500afe6b2025f9c46846e12" + }, + { + "name": "Mexican McAloo Tikki Burger Combo", + "price": "223.99", + "description": "Enjoy a delicious combo of Mexican McAloo Tikki Burger + Fries (M) + Beverage of your choice in a new, delivery friendly, reusable bottle..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/0140c49c7274cdb6af08053af1e6cc20" + }, + { + "name": "McEgg Burger Combo", + "price": "230.99", + "description": "Enjoy a combo of McEgg + Fries (M) + Drink of your Choice . Order now to experience a customizable, delicious meal..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/4f474c833930fa31d08ad2feed3414d8" + }, + { + "name": "McChicken Burger Combo", + "price": "314.99", + "description": "Your favorite McChicken Burger + Fries (M) + Drink of your choice in a new, delivery friendly, resuable bottle..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/b943fe56-212d-4366-a085-4e24d3532b30_3776df33-e18d-46c9-a566-c697897f1d16.png" + }, + { + "name": "McSpicy Chicken Burger Combo", + "price": "363.99", + "description": "Your favorite McSpicy Chicken Burger + Fries (M) + Drink of your choice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/214189a0-fd53-4f83-9d18-c3e53f2c017a_70748bf8-e587-47d6-886c-9b7ac57e84b3.png" + }, + { + "name": "McSpicy Paneer Burger Combo", + "price": "344.99", + "description": "Enjoy your favourite McSpicy Paneer Burger + Fries (M) + Drink of your Choice . Order now to experience a customizable, delicious combo.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/70fc7aa0-7f3b-418e-8ccd-5947b5f1aacd_055bbe77-fbe7-4200-84d3-4349475eb298.png" + }, + { + "name": "Veg Maharaja Mac Burger Combo", + "price": "398.99", + "description": "Enjoy a double decker Veg Maharaja Mac+ Fries (M) + Drink of your Choice . Order now to experience a customizable, delicious meal..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/d064fb5e-fb2e-4e1d-9515-18f26489f5b1_f6f49dba-2ae7-4ebf-8777-548c5ad4d799.png" + }, + { + "name": "McVeggie Burger Combo", + "price": "308.99", + "description": "Enjoy a combo of McVeggie + Fries (M) + Drink of your Choice in a new, delivery friendly, resuable bottle..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/5c9942c5-bc9d-4637-a312-cf51fe1d7aa8_e7e13473-9424-4eb8-87c3-368cfd084a2f.png" + }, + { + "name": "Big Spicy Chicken Wrap Combo", + "price": "379.99", + "description": "Your favorite Big Spicy Chicken Wrap + Fries (M) + Drink of your choice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/22/f631f834-aec2-410d-ab9d-7636cd53d7a0_8801a3c7-ad83-4f7c-bcb0-76101fadde91.png" + }, + { + "name": "Chicken Maharaja Mac Burger Combo", + "price": "398.99", + "description": "Enjoy a double decker Chicken Maharaja Mac + Fries (M) + Drink of your Choice . Order now to experience a customizable, delicious meal..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/ead090f2-5f80-4159-a335-d32658bcfc7c_8bcc5cd9-b22a-4f5d-8cde-0a2372c985c8.png" + }, + { + "name": "Grilled Cheese and Chicken Burger Combo", + "price": "310.47", + "description": "Enjoy a combo of Grilled Chicken & Cheese Burger + Fries (M) + Coke . Order now to experience a customizable, delicious meal..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/f6813404-54a4-492b-a03a-cf32d00ee1ae_c98c4761-69eb-4503-bde3-dffaafd43b15.png" + }, + { + "name": "Corn & cheese Burger Combo", + "price": "310.47", + "description": "Enjoy a combo of Corn & Cheese Burger + Fries (M) + Coke . Order now to experience a customizable, delicious meal..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/c8839f22-e525-44bb-8a50-8ee7cffecd26_cf4c0bcf-c1b9-4c0e-9109-03eb86abf4dd.png" + }, + { + "name": "Birthday Party Package - McChicken", + "price": "2169.14", + "description": "5 McChicken Burger + 5 Sweet Corn + 5 B Natural Mixed Fruit Beverage + 5 Soft Serve (M) + Book.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/15/37e0bc09-3e46-41d6-8146-66d6962ae2ab_0c91e294-cad2-4177-a2a9-86d6523bb811.png" + }, + { + "name": "Birthday Party Package - McVeggie", + "price": "2169.14", + "description": "5 McVeggie Burger + 5 Sweet Corn + 5 B Natural Mixed Fruit Beverage + 5 Soft Serve (M) + Book.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/15/0be6390f-e023-47f8-8bf4-9bb16d4995ce_2eceb8c2-6e90-47d7-97d6-93960391e668.png" + }, + { + "name": "McEgg Burger Happy Meal", + "price": "231.42", + "description": "Enjoy a combo of McEgg Burger + Sweet Corn + B Natural Mixed Fruit Beverage + Book.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/ada14e04-680d-44ef-bb05-180d0cc26ebc_b7bda720-9fcd-4bf4-97eb-bfb5a98e0239.png" + }, + { + "name": "McCheese Burger Veg Combo", + "price": "388.99", + "description": "Enjoy a deliciously filling meal of McCheese Veg Burger + Fries (M) + Beverage of your Choice in a delivery friendly, reusable bottle..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/22b25dd0-80ed-426b-a323-82c6b947612d_df827568-54f9-4329-af60-1fff38d105e9.png" + }, + { + "name": "McSpicy Premium Burger Chicken Combo", + "price": "398.99", + "description": "A deliciously filling meal of McSpicy Premium Chicken Burger + Fries (M) + Drink of your choice.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/28550356-fea3-47ce-bfa8-11222c78958a_8975c012-a121-42c3-92e7-719644761d83.png" + }, + { + "name": "McSpicy Premium Burger Veg Combo", + "price": "384.99", + "description": "A deliciously filling meal of McSpicy Premium Veg Burger + Fries (M) + Drink of your choice.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/14ce2cab-913c-4916-b98a-df2e5ea838ff_6d6208ca-4b76-4b87-a3fe-72c2db1081d8.png" + }, + { + "name": "McCheese Burger Chicken Combo", + "price": "388.99", + "description": "Enjoy a deliciously filling meal of McCheese Chicken Burger + Fries (M) + Beverage of your Choice in a delivery friendly, reusable bottle..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/b32eb9b3-7873-4f00-b601-0c92dd5a71ef_ec318a73-0e41-4277-8606-dc5c33b04533.png" + }, + { + "name": "McAloo Tikki Burger Combo", + "price": "204.99", + "description": "Enjoy a delicious combo of McAloo Tikki Burger + Fries (M) + Beverage of your choice in a new, delivery friendly, reusable bottle..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/b03a3ad7212fca0da40e90eed372ced9" + }, + { + "name": "McCheese Burger Veg Combo with Corn", + "price": "415.99", + "description": "Enjoy a combo of McCheese Burger Veg, Classic corn, McFlurry Oreo (Small) with a beverage of your choice in a delivery friendly, resuable bottle..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/9068eeb6-c774-4354-8c18-bf2ddcc94d10_631c39d1-d78c-4275-a14f-60670ce5aee4.png" + }, + { + "name": "2 Pc Chicken Nuggets Happy Meal", + "price": "211.42", + "description": "Enjoy a combo of 2 Pc Chicken Nuggets + Sweet Corn+ B Natural Mixed Fruit Beverage + Book.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/ac8a1d83-59c4-438f-bccb-edd601dacbf5_872c3881-77a1-43f7-82ca-2a929886045d.png" + }, + { + "name": "4 Pc Chicken Nuggets Happy Meal", + "price": "259.42", + "description": "Enjoy a combo of 4 Pc Chicken Nuggets + Sweet Corn+ B Natural Mixed Fruit Beverage + Book.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/fab04bb4-4f67-459a-85ef-5e9ae7861c88_f6761c27-26b3-4456-8abb-8f6c54274b34.png" + }, + { + "name": "Chicken McNuggets 6 Pcs Combo", + "price": "350.99", + "description": "Enjoy your favorite Chicken McNuggets + Fries (M) + Drink of your choice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6e7f9411ed67fe8d8873734af1e8d4e9" + }, + { + "name": "Chicken Surprise Burger + 4 Pc Chicken McNuggets + Coke", + "price": "259.99", + "description": "Enjoy the newly launched Chicken Surprise Burger with 4 Pc Chicken McNuggets and Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/4/6c0288a4-943a-4bb6-a810-b14877c0ea8f_f56187da-4ae4-4a88-b1a5-e48e28d78087.png" + }, + { + "name": "Chicken Surprise Burger Combo", + "price": "238.09", + "description": "Chicken Surprise Burger + Fries (M) + Drink of your choice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/cb9833dd-5983-4445-bb1b-8d0e70b6c930_8928ae23-ddae-4d9a-abc5-e887d7d7868e.png" + }, + { + "name": "Crispy Veggie Burger Meal (M)", + "price": "326.99", + "description": "A flavorful patty with 7 premium veggies, zesty cocktail sauce, and soft buns, paired with crispy fries (M) and a refreshing Coke (M). A perfectly satisfying and full-flavored meal!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/eb4650ba-e950-4953-b1a7-c03a691ac0d2_6ca20c30-4c62-462a-b46d-9d1f21786b49.png" + }, + { + "name": "Mc Crispy Chicken Burger Meal (M)", + "price": "366.99", + "description": "A crunchy, golden chicken thigh fillet with fresh lettuce and creamy pepper mayo between soft, toasted premium buns, served with crispy fries (M) and a refreshing Coke (M). A perfectly satisfying and full-flavored meal!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/eedf0540-f558-4945-b996-994b1cd58048_d9af8cc8-384e-48c3-ab59-2facadfe574a.png" + }, + { + "name": "Choco Crunch Cookie + McAloo Tikki Burger + Lemon Ice Tea", + "price": "284.76", + "description": "Indulge in the perfect combo,crispy Choco Crunch Cookie, classic Aloo Tikki Burger, and refreshing Lemon Iced Tea. A delicious treat for your cravings, delivered fresh to your doorstep!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/b8ed02df-fb8f-4406-a443-398731aa9ef3_dafd8af6-e740-4987-8f4a-7bdc51dda4d8.png" + }, + { + "name": "Veg Pizza McPuff + Choco Crunch Cookie + Americano", + "price": "284.76", + "description": "A delightful trio, savoury Veg Pizza McPuff, crunchy Choco Crunch Cookie, and bold Americano, perfect for a satisfying snack break!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/9d81dad2-06a4-4402-9b07-2ed418963e16_f256697b-a444-4dfa-a9ff-d6448bb67b4b.png" + }, + { + "name": "Big Yummy Cheese Meal (M)", + "price": "424.76", + "description": "Double the indulgence, double the flavor: our Big Yummy Cheese Burger meal layers a spicy paneer patty and Cheese patty with crisp lettuce and smoky chipotle sauce, served with fries (M) and a beverage of your choice.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/cb892b9e-48b5-4127-b84f-b5e395961ddf_755bf2c0-e30a-4185-81e7-c35cbd07f483.png" + }, + { + "name": "Big Yummy Chicken Meal (M)", + "price": "424.76", + "description": "Indulge in double the delight: our Big Yummy Chicken Burger meal pairs the tender grilled chicken patty and Crispy chicken patty with crisp lettuce, jalapeños, and bold chipotle sauce, served with fries (M) and a beverage of your choice ..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/93313e0c-da3b-4490-839e-fd86b84c2644_b0076aa7-ef47-4f13-b72a-40f64744db95.png" + }, + { + "name": "McSpicy Chicken Double Patty Burger", + "price": "278.19", + "description": "Indulge in our signature tender double chicken patty, coated in spicy, crispy batter, topped with creamy sauce, and crispy lettuce.. Contains: Soybeans, Milk, Egg, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/314b5b5786f73746de4880602723a913" + }, + { + "name": "McChicken Double Patty Burger", + "price": "173.24", + "description": "Enjoy the classic, tender double chicken patty with creamy mayonnaise and lettuce in every bite. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/af88f46a82ef5e6a0feece86c349bb00" + }, + { + "name": "McVeggie Double Patty Burger", + "price": "186.12", + "description": "Savour your favorite spiced double veggie patty, lettuce, mayo, between toasted sesame buns in every bite. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/2d5062832f4d36c90e7dfe61ef48e85a" + }, + { + "name": "Mexican McAloo Tikki Double Patty Burger", + "price": "93.05", + "description": "A fusion of International taste combined with your favourite aloo tikki now with two patties. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/cda0e2d51420a95fad28ad728914b6de" + }, + { + "name": "McAloo Tikki Double Patty Burger", + "price": "88.11", + "description": "The World's favourite Indian burger! A crispy double Aloo patty, tomato mayo sauce & onions. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/ef569f74786e6344883a1decdd193229" + }, + { + "name": "McAloo Tikki Burger", + "price": "69.30", + "description": "The World's favourite Indian burger! A crispy Aloo patty, tomato mayo sauce & onions. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/b13811eeee71e578bc6ca89eca0ec87f" + }, + { + "name": "Big Spicy Paneer Wrap", + "price": "239.58", + "description": "Rich & filling cottage cheese patty coated in spicy crispy batter, topped with tom mayo sauce wrapped with lettuce, onions, tomatoes & cheese.. Contains: Sulphite, Soybeans, Peanut, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/22/198c3d14-3ce8-4105-8280-21577c26e944_779c4353-2f85-4515-94d2-208e90b830eb.png" + }, + { + "name": "McSpicy Chicken Burger", + "price": "226.71", + "description": "Indulge in our signature tender chicken patty, coated in spicy, crispy batter, topped with creamy sauce, and crispy lettuce.. Contains: Soybeans, Egg, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/dcdb436c-7b9f-4667-9b73-b8fa3215d7e2_9730340a-661b-49f4-a7d9-a8a89ffe988f.png" + }, + { + "name": "McSpicy Paneer Burger", + "price": "225.72", + "description": "Indulge in rich & filling spicy paneer patty served with creamy sauce, and crispy lettuce—irresistibly satisfying!. Contains: Sulphite, Soybeans, Peanut, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/fb912d21-ad9d-4332-b7cf-8f65d69e2c47_1fa5998c-b486-449a-9a88-2e61cf92ff77.png" + }, + { + "name": "Mexican McAloo Tikki Burger", + "price": "75.42", + "description": "Your favourite McAloo Tikki with a fusion spin with a Chipotle sauce & onions. Contains: Sulphite, Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/167aeccf27bab14940fa646c8328b1b4" + }, + { + "name": "McVeggie Burger", + "price": "153.44", + "description": "Savour your favorite spiced veggie patty, lettuce, mayo, between toasted sesame buns in every bite. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/2cf63c01-fef1-49b6-af70-d028bc79be7b_bfe88b73-33a9-489c-97f3-fb24631de1fc.png" + }, + { + "name": "McEgg Burger", + "price": "69.30", + "description": "A steamed egg, spicy Habanero sauce, & onions on toasted buns, a protein packed delight!. Contains: Soybeans, Milk, Egg, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/265c57f68b1a52f1cc4b63acf082d611" + }, + { + "name": "Veg Maharaja Mac Burger", + "price": "246.51", + "description": "Savor our filling 11 layer burger! Double the indulgence with 2 corn & cheese patties, along with jalapeños, onion, cheese, tomatoes, lettuce, and spicy Cocktail sauce. . Contains: Sulphite, Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/06354d09-be1b-406c-86b5-49dc9b5062d1_2f80f39e-c951-4ca6-8fca-a243a18c3448.png" + }, + { + "name": "McChicken Burger", + "price": "150.47", + "description": "Enjoy the classic, tender chicken patty with creamy mayonnaise and lettuce in every bite. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/c093ba63-c4fe-403e-811a-dc5da0fa6661_f2ad8e1e-5162-4cf8-8dab-5cc5208cdb85.png" + }, + { + "name": "Grilled Chicken & Cheese Burger", + "price": "172.25", + "description": "A grilled chicken patty, topped with sliced cheese, spicy Habanero sauce, with some heat from jalapenos & crunch from onions. Contains: Sulphite, Soybeans, Egg, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/55a77d9e-cc28-4853-89c8-1ba3861f38c4_a378aafc-62b3-4328-a255-0f35f810966e.png" + }, + { + "name": "Corn & Cheese Burger", + "price": "166.32", + "description": "A juicy corn and cheese patty, topped with extra cheese, Cocktail sauce, with some heat from jalapenos & crunch from onions. Contains: Sulphite, Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/cb4d60c0-72c0-4694-8d41-3c745e253ea6_8262ec8c-e2ac-4c4f-8e52-36144a372851.png" + }, + { + "name": "Big Spicy Chicken Wrap", + "price": "240.57", + "description": "Tender and juicy chicken patty coated in spicy, crispy batter, topped with a creamy sauce, wrapped with lettuce, onions, tomatoes & cheese. A BIG indulgence.. Contains: Soybeans, Egg, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/22/e1fa4587-23ac-4613-af61-de659b066d19_14205d12-ea39-4894-aa85-59035020cecd.png" + }, + { + "name": "McCheese Burger Chicken", + "price": "282.15", + "description": "Double the indulgence with a sinfully oozing cheesy patty & flame-grilled chicken patty, along with chipotle sauce, shredded onion, jalapenos & lettuce.. Contains: Sulphite, Soybeans, Egg, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/b1184d5f-0785-4393-98a8-a712d280a045_027d0e63-e5d9-43f3-9c9c-0dc68dd1ece1.png" + }, + { + "name": "McCheese Burger Veg", + "price": "262.35", + "description": "Find pure indulgence in our Veg McCheese Burger, featuring a sinfully oozing cheesy veg patty, roasted chipotle sauce, jalapenos & lettuce.. Contains: Sulphite, Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/de84d4cc-6169-4235-942e-e4883a81c2e0_d90762c6-283f-46e5-a6ed-14ee3262bae0.png" + }, + { + "name": "McSpicy Premium Chicken Burger", + "price": "259.38", + "description": "A wholesome Spicy Chicken patty, Lettuce topped with Jalapenos and Cheese slice, Spicy Cocktail sauce & Cheese sauce. Contains: Sulphite, Soybeans, Egg, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/e1c10ab2-671b-4eac-aee8-88d9f96e005b_40169ccb-b849-4e0d-a54a-8938dd41ea34.png" + }, + { + "name": "McSpicy Premium Veg Burger", + "price": "249.47", + "description": "A wholesome Spicy Paneer patty, Lettuce topped with Jalapenos and Cheese slice, Spicy Cocktail sauce & Cheese sauce. Contains: Sulphite, Soybeans, Peanut, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/1a9faafd-b523-40a2-bdc6-f35191cfcf4a_0c462cb4-3843-4997-b813-dc34249b7c91.png" + }, + { + "name": "Chicken Maharaja Mac Burger", + "price": "268.28", + "description": "Savor our filling 11 layer burger! Double the indulgence with 2 juicy grilled chicken patties, along with jalapeños, onion, cheese, tomatoes, lettuce, and zesty Habanero sauce. . Contains: Sulphite, Soybeans, Egg, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/65a88cf4-7bcd-40f6-a09d-ec38c307c5d9_8993ea5b-f8e0-4d6c-9691-7f85adee2000.png" + }, + { + "name": "Chicken Surprise Burger", + "price": "75.23", + "description": "Introducing the new Chicken Surprise Burger which has the perfect balance of a crispy fried chicken patty, the crunch of onions and the richness of creamy sauce.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/0fbf18a1-5191-4cda-a09d-521a24c8c6ca_25cf57c6-48cc-47bd-b422-17e86b816422.png" + }, + { + "name": "McAloo Tikki Burger NONG", + "price": "69.30", + "description": "The World's favourite Indian burger with No Onion & No Garlic! Crispy aloo patty with delicious Tomato Mayo sauce!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/60311aec-07af-483d-b66a-ecae76edbd75_1407e287-7f07-4717-a6b1-14bff1a34961.png" + }, + { + "name": "Mexican McAloo Tikki Burger NONG", + "price": "74.24", + "description": "Your favourite McAloo Tikki with a fusion spin of Chipotle sauce. No Onion and No Garlic.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/c7d3076a-dfa7-4725-89cd-53754cecebee_1c11b7da-e00d-4e6b-bbe8-8ee847ee88a1.png" + }, + { + "name": "Crispy Veggie Burger", + "price": "198", + "description": "A flavorful patty made with a blend of 7 premium veggies, topped with zesty cocktail sauce, all served between soft, premium buns. Perfectly satisfying and full of flavor.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/7a3244bf-3091-4ae6-92e3-13be841a753e_b21f7a05-24b3-43f8-a592-beb23e6b69fa.png" + }, + { + "name": "Mc Crispy Chicken Burger", + "price": "221.76", + "description": "A crunchy, golden chicken thigh fillet, topped with fresh lettuce and creamy pepper mayo, all nestled between soft, toasted premium buns. Perfectly satisfying and full of flavor.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/df040551-263c-4074-86ee-68cb8cd393ba_3a5e53c6-601e-44a8-bf2e-590bffd7ee5e.png" + }, + { + "name": "McAloo Tikki Burger with Cheese", + "price": "98.05", + "description": "Savor the classic McAloo Tikki Burger, with an add-on cheese slice for a cheesy indulgence.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/10/30/0f0cbfd7-ee83-4e7c-bb6d-baab81e51646_4746c010-f82c-4e23-a43f-e36fe50b883b.png" + }, + { + "name": "Mexican McAloo Tikki with Cheese", + "price": "98.05", + "description": "Savor your favourite Mexican McAloo Tikki Burger, with an add-on cheese slice for a cheesy indulgence.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/10/30/d3cfbc30-4a7c-40a8-88a9-f714f3475523_14e7323b-24fe-4fbf-a517-68de64489103.png" + }, + { + "name": "Big Yummy Cheese Burger.", + "price": "349", + "description": "A spicy, cheesy indulgence, the Big Yummy Cheese Burger stacks a fiery paneer patty and a rich McCheese patty with crisp lettuce and smoky chipotle sauce on a Quarter Pound bun.. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/5c65bd4d-86ff-424c-800c-7d8b04aac86d_99e0d34c-cbba-41d9-bc31-a093237ba2af.png" + }, + { + "name": "Big Yummy Chicken Burger.", + "price": "349", + "description": "Crafted for true indulgence, tender grilled chicken patty meets the McCrispy chicken patty, elevated with crisp lettuce, jalapenos, and bold chipotle sauce.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/33234ab9-f10d-4605-89fc-349e0c7058bf_edba8f21-7c28-4fb6-88a2-dd3a86966175.png" + }, + { + "name": "Fries (Regular)", + "price": "88.11", + "description": "World Famous Fries, crispy, golden, lightly salted and fried to perfection! Also known as happiness.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/5a18fbbff67076c9a4457a6b220a55d9" + }, + { + "name": "Fries (Large)", + "price": "140.58", + "description": "World Famous Fries, crispy, golden, lightly salted and fried to perfection! Also known as happiness.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/a4b3002d0ea35bde5e5983f40e4ebfb4" + }, + { + "name": "Tomato Ketchup Sachet", + "price": "1", + "description": "Looking for a sauce to complement your meal? Look no further.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7db5533db29a4e9d2cc033f35c5572bc" + }, + { + "name": "9 Pc Chicken Nuggets", + "price": "221.74", + "description": "9 pieces of our iconic crispy, golden fried Chicken McNuggets!. Contains: Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/1ca7abb262e8880f5cb545d0d2f9bb9b" + }, + { + "name": "6 Pc Chicken Nuggets", + "price": "183.14", + "description": "6 pieces of our iconic crispy, golden fried Chicken McNuggets!. Contains: Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/44dc10c1099d7c366db9f5ce776878bd" + }, + { + "name": "Piri Piri Spice Mix", + "price": "23.80", + "description": "The perfect, taste bud tingling partner for our World Famous Fries. Shake Shake, and dive in!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/df3edfc74f610edff535324cc53a362a" + }, + { + "name": "Fries (Medium)", + "price": "120.78", + "description": "World Famous Fries, crispy, golden, lightly salted and fried to perfection! Also known as happiness.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/8a61e7fd97c454ea14d0750859fcebb8" + }, + { + "name": "Chilli Sauce Sachet", + "price": "2", + "description": "Looking for a sauce to complement your meal? Look no further.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f708dfc29c9624d8aef6e6ec30bde1c9" + }, + { + "name": "Veg Pizza McPuff", + "price": "64.35", + "description": "Crispy brown crust with a generous filling of rich tomato sauce, mixed with carrots, bell peppers, beans, onions and mozzarella. Served HOT.. Contains: Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/abe4b8cdf0f1bbfd1b9a7a05be3413e8" + }, + { + "name": "Classic Corn Cup", + "price": "90.08", + "description": "A delicious side of golden sweet kernels of corn in a cup.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9d67eae020425c4413acaf5af2a29dce" + }, + { + "name": "Fries (M) + Piri Piri Mix", + "price": "125", + "description": "Flat 15% Off on Fries (M) + Piri Piri Mix.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/15/b02b9b46-0d2b-46a6-aaa3-171c35101e11_24766c28-4fe9-4562-92d3-85c39d29c132.png" + }, + { + "name": "Cheesy Fries", + "price": "157.40", + "description": "The world famous, crispy golden Fries, served with delicious cheese sauce with a hint of spice. Contains cheese & mayonnaise. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/086cb28d-501d-42e5-a603-33e2d4493588_11348186-570f-44b8-b24e-88855455ba25.png" + }, + { + "name": "20 Pc Chicken Nuggets", + "price": "445.97", + "description": "20 pieces of our iconic crispy, golden fried Chicken McNuggets!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/1ca7abb262e8880f5cb545d0d2f9bb9b" + }, + { + "name": "Barbeque Sauce", + "price": "19.04", + "description": "Looking for a sauce to complement your meal? Look no further.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/ba0a188d45aecc3d4d187f340ea9df54" + }, + { + "name": "4 Pcs Chicken Nuggets", + "price": "109.88", + "description": "4 pieces of our iconic crispy, golden fried Chicken McNuggets!. Contains: Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/44dc10c1099d7c366db9f5ce776878bd" + }, + { + "name": "Mustard Sauce", + "price": "19.04", + "description": "Looking for a sauce to complement your meal? Look no further.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6c3aeffdbd544ea3ceae1e4b8ce3fc43" + }, + { + "name": "Spicy Sauce", + "price": "33.33", + "description": "Enjoy this spicy sauce that will add an extra kick to all your favourite items.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/dcc9ef6b-ceda-4b15-87af-ba2ad6c7de28_cfa5487a-811c-4c41-a770-af4ba6c5ebc8.png" + }, + { + "name": "Mango Smoothie", + "price": "210.86", + "description": "A delicious mix of mangoes, soft serve mix and blended ice. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/bb8470e1-5862-4844-bb47-ec0b2abdd752_845f1527-6bb0-46be-a1be-2fcc7532e813.png" + }, + { + "name": "Strawberry Green Tea (S)", + "price": "153.44", + "description": "Freshly-brewed refreshing tea with fruity Strawbery flavour.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/167dc0134bc1f4e8d7cb8e5c2a9dde5d" + }, + { + "name": "Moroccan Mint Green Tea (R )", + "price": "203.94", + "description": "Freshly-brewed refreshing tea with hint of Moroccon Mint flavour.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/2699c7c2130b4f50a09af3e294966b2e" + }, + { + "name": "Americano Coffee (R)", + "price": "190.07", + "description": "Refreshing cup of bold and robust espresso made with our signature 100% Arabica beans, combined with hot water.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/6a300499-efad-4a89-8fe4-ef6f6d3c1e2d_db248587-e3ea-4034-bbd7-33f9e483d6cb.png" + }, + { + "name": "Mixed Berry Smoothie", + "price": "210.86", + "description": "A mix of mixed berries, blended together with our creamy soft serve. Contains: Sulphite, Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/7a70ab81-ef1e-4e13-9c78-43ed2c62eaeb_3cd64c86-3a7b-41a6-9528-5fb5e2bbadc7.png" + }, + { + "name": "McCafe-Ice Coffee", + "price": "202.95", + "description": "Classic coffee poured over ice with soft servefor a refreshing pick-me-up. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/14/fd328038-532b-4df7-b6e9-b21bd6e8c70f_7e663943-ab8a-4f24-8dc0-d727dd503cd3.png" + }, + { + "name": "American Mud Pie Shake", + "price": "202.95", + "description": "Creamy and rich with chocolate and blended with nutty brownie bits for that extra thick goodness. Contains: Soybeans, Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/5ad3343d-a520-498d-a933-52104c624304_9f6cb499-0229-40dc-8b17-4c386f0cc287.png" + }, + { + "name": "Ice Americano Coffee", + "price": "180.18", + "description": "Signature Arabica espresso shot mixed with ice for an energizing experience.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7d89db9d67c537d666d838ddc1e0c44f" + }, + { + "name": "Americano Coffee (S)", + "price": "170.27", + "description": "Refreshing cup of bold and robust espresso made with our signature 100% Arabica beans, combined with hot water.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/6a300499-efad-4a89-8fe4-ef6f6d3c1e2d_db248587-e3ea-4034-bbd7-33f9e483d6cb.png" + }, + { + "name": "Coke", + "price": "101.97", + "description": "The perfect companion to your burger, fries and everything nice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/a1afed29afd8a2433b25cc47b83d01da" + }, + { + "name": "Mocha Coffee (R)", + "price": "236.60", + "description": "A delight of ground Arabica espresso, chocolate syrup and steamed milk. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/04a137420bf3febf861c4beed86d5702" + }, + { + "name": "Mocha Coffee (S)", + "price": "210.86", + "description": "A delight of ground Arabica espresso, chocolate syrup and steamed milk. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/71df07eb87d96824e2122f3412c8f743" + }, + { + "name": "Hot Chocolate (R)", + "price": "224.73", + "description": "Sinfully creamy chocolate whisked with silky streamed milk. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/1948679f-65df-4f93-a9cc-2ec796ca0818_6ad8b55d-e3cc-48ed-867d-511100b5735d.png" + }, + { + "name": "Latte Coffee (R)", + "price": "202.95", + "description": "A classic combination of the signature McCafe espresso, smooth milk, steamed and frothed. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/cee1ec0e10e25018572adcaf3a3c9e8c" + }, + { + "name": "Coke zero can", + "price": "66.66", + "description": "The perfect diet companion to your burger, fries and everything nice. Regular serving size, 300 Ml..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/8d6a37c4fc69bceb66b6a66690097190" + }, + { + "name": "Schweppes Water bottle", + "price": "66.66", + "description": "Quench your thirst with the Schweppes Water bottle.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/6/df44cf7c-aa13-46e2-9b38-e95a2f9faed4_d6c65089-cd93-473b-b711-aeac7fcf58b0.png" + }, + { + "name": "Fanta", + "price": "101.97", + "description": "Add a zest of refreshing orange to your meal..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7d662c96bc13c4ac33cea70c691f7f28" + }, + { + "name": "Mixed Fruit Beverage", + "price": "76.19", + "description": "Made with puree, pulp & juice from 6 delicious fruits. Contains: Soybeans, Peanut, Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/8e455d39bbbd8e4107b2099da51f3933" + }, + { + "name": "Sprite", + "price": "101.97", + "description": "The perfect companion to your burger, fries and everything nice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/46e03daf797857bfbce9f9fbb539a6aa" + }, + { + "name": "Cappuccino Coffee (R)", + "price": "199.98", + "description": "A refreshing espresso shot of 100% Arabica beans, topped with steamed milk froth. 473ml. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/4f45f0d8-111a-4d9c-a993-06d6858fdb06_cb5d79e9-c112-45e5-9e6a-e17d75acd8ac.png" + }, + { + "name": "Cappuccino Coffee (S)", + "price": "170.27", + "description": "A refreshing espresso shot of 100% Arabica beans, topped with steamed milk froth. 236ml. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/4f45f0d8-111a-4d9c-a993-06d6858fdb06_cb5d79e9-c112-45e5-9e6a-e17d75acd8ac.png" + }, + { + "name": "Berry Lemonade Regular", + "price": "141.57", + "description": "A refreshing drink, made with the delicious flavors of berries. 354 ml..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/14/4f0f852a-1e4a-4bd9-b0ec-69ccef3c6a34_0f4f0b64-1b1b-4a7d-98f9-e074fc96d1bd.png" + }, + { + "name": "Chocolate Flavoured Shake", + "price": "183.15", + "description": "The classic sinful Chocolate Flavoured Shake, a treat for anytime you need one. Now in new, convenient and delivery friendly packaging. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/463331b6-aa39-4d37-a8b2-c175aa20f723_14e888b6-2ebf-4d84-93a7-7b0576147bda.png" + }, + { + "name": "McCafe-Classic Coffee", + "price": "214.82", + "description": "An irrestible blend of our signature espresso and soft serve with whipped cream on top, a timeless combination! Now in a new, convenient and delivery friendly packaging.. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/5f9bdb36689a11cadb601a27b6fdef2d" + }, + { + "name": "Mocha Frappe", + "price": "278.19", + "description": "The perfect mix of indulgence and bold flavours. Enjoy a delicious blend of coffee, chocolate sauce, and soft serve. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/063e9cd747c621978ab4fddbb6d0a5ee" + }, + { + "name": "Cappuccino Small with Hazelnut", + "price": "183.15", + "description": "A delightful and aromatic coffee beverage that combines the robust flavor of espresso with the rich, nutty essence of hazelnut.. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/1793c61c-48f4-4785-b8a6-bc50eadb3b88_36ab0b13-0b17-459c-97a6-50dea3027b80.png" + }, + { + "name": "Ice Tea - Green Apple flavour", + "price": "182.16", + "description": "A perfect blend of aromatic teas, infused with green apple flavour .Now in a new, convenient and delivery friendly packaging.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/54243043b1b31fa715f38e6998a63e93" + }, + { + "name": "Strawberry Shake", + "price": "183.15", + "description": "An all time favourite treat bringing together the perfect blend of creamy vanilla soft serve and strawberry flavor.Now in new, convenient and delivery friendly packaging. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/b7d85483-4328-40a7-8cba-c408771d2482_99cab512-28fc-4b2c-b07f-87850002e79c.png" + }, + { + "name": "Cappuccino Small with French Vanilla", + "price": "182.16", + "description": "A popular coffee beverage that combines the smooth, creamy flavor of vanilla with the robust taste of espresso. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/e0a5c619-2df5-431f-9e21-670a358e8dbb_c2a2324f-ff95-44b3-adf9-dd6aee9696aa.png" + }, + { + "name": "Classic Coffee Regular with French Vanilla", + "price": "236", + "description": "a delightful and refreshing beverage that blends into the smooth, creamy essence of vanilla with the invigorating taste of chilled coffee.. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/5926eaba-bd6a-4af1-ba57-89bdb2fdfec6_c2507538-7597-4fc0-92e3-bdf57c81bda5.png" + }, + { + "name": "Classic Coffee Regular with Hazelnut", + "price": "233.63", + "description": "refreshing and delicious beverage that combines the rich, nutty taste of hazelnut with the cool, invigorating essence of cold coffee. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/b65446d8-bee6-484c-abdf-0326315c7e40_4c694dbc-e5e9-4998-b6ac-bd280fc6b5dc.png" + }, + { + "name": "Iced Coffee with French Vanilla", + "price": "223.74", + "description": "An ideal choice for those who enjoy a smooth, creamy vanilla twist to their iced coffee, providing a satisfying and refreshing pick-me-up. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/35867352-7802-4082-8b25-955878a6daa2_1d995db8-655f-45d3-9ad2-4469fe1301ae.png" + }, + { + "name": "Iced Coffee with Hazelnut", + "price": "223.74", + "description": "An ideal choice for those who enjoy a flavorful, nutty twist to their iced coffee, providing a satisfying and refreshing pick-me-up.. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/0619409e-e7cc-4b7c-a002-c05a2dff2ed8_f9bf368f-3dae-4af5-813d-4645384d24f7.png" + }, + { + "name": "Mint Lime Cooler", + "price": "101.97", + "description": "Refresh your senses with our invigorating Mint Lime Cooler. This revitalizing drink combines the sweetness of fresh lime juice and the subtle tang, perfectly balanced to quench your thirst and leave you feeling revitalized. Contains: Gluten, Milk, Peanut, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/a6b79a50-55c4-4b83-8b3c-d59c86ee1337_a3063ad4-fee2-4926-bf6d-ec98051b2249.png" + }, + { + "name": "Choco Crunch Cookie", + "price": "95", + "description": "Grab the choco crunch cookies packed with chocolate chips for the perfect crunch. Contains: Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/32ff88484a8607b6d740c1635b9ce09f" + }, + { + "name": "Hot Coffee Combo", + "price": "181", + "description": "In this combo choose any 1 hot coffee among- Cappuccino(s)/Latte(s)/Mocha(s)/Americano(s) and any 1 snacking item among- Choco crunch cookies/Oats Cookies/Choco Brownie/Blueberry muffin/ Signature croissant.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9aba6528e9c4f780dda18b5068349020" + }, + { + "name": "Indulge Choco Jar Dessert", + "price": "76", + "description": "Rich chocolate for pure indulgence to satify your sweet tooth..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/c7165efd872543e1648c21c930dafe5f" + }, + { + "name": "Cinnamon Raisin Cookie", + "price": "95", + "description": "Enjoy the wholesome flavours of this chewy and satisfying cookie. Contains: Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/fa5526832dfc6e236e6de7c322beae94" + }, + { + "name": "Cold Coffee Combo", + "price": "185", + "description": "In this combo choose any 1 coffee among- Cold Coffee ( R )/Iced Coffee ( R )/Iced Americano ( R ) and any 1 snacking item among- Choco crunch cookies/Oats Cookies/Choco Brownie/Blueberry muffin.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/cb24719daf96b58dd1be3bbf7b9fe372" + }, + { + "name": "Chocochip Muffin", + "price": "142", + "description": "Enjoy a dense chocochip muffin, with melty chocolate chips for a choco-lover's delight. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6c748c0b21f4a99593d1040021500430" + }, + { + "name": "Indulge Combo", + "price": "171", + "description": "Indulge in the perfect pairing of a classic cold coffee and a chocolate chip muffin, that balances the refreshing taste of chilled coffee with the sweet, comforting flavors of a freshly baked treat..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/ee2c7b9f-4426-4e70-9e98-bf8e8cc754ad_93eaaaa3-e498-4ca1-a77d-545a8006c8e2.png" + }, + { + "name": "Take a break Combo", + "price": "171", + "description": "Savor the harmonious pairing of a classic cappuccino with a cinnamon cookie, combining the bold, creamy flavors of coffee with the warm, spiced sweetness of a baked treat .", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/87a8807b-77e4-4e0d-a47e-af1cf2d2b96d_3230e601-af8f-4436-85cd-14c4e855240d.png" + }, + { + "name": "Treat Combo", + "price": "171", + "description": "Delight in the luxurious pairing of a chocolate jar dessert with a classic cappuccino, combining rich, creamy indulgence with the bold, aromatic flavors of espresso..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/9462ac0c-840e-44d1-a85d-bbea6f8268f1_a93dc44b-bf30-47b9-8cee-3a5589d2002e.png" + }, + { + "name": "Butter Croissant", + "price": "139", + "description": "Buttery, flaky croissant baked to golden perfection.Light, airy layers with a crisp outer shell.A classic French treat that melts in your mouth..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/da6cbf69-e422-4d02-9cc9-cb0452d04874_2153fe0c-389e-4842-b4c9-e0b78c450625.png" + }, + { + "name": "Butter Croissant + Cappuccino", + "price": "209", + "description": "Buttery croissant paired with a rich, frothy cappuccino.Warm, comforting, and perfectly balanced.A timeless duo for your anytime cravings..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/227a6aa3-f18c-4685-bb09-a76a01ec39a1_f347d737-b634-43d7-9b73-dc99bccde65f.png" + }, + { + "name": "Butter Croissant + Iced Coffee", + "price": "209", + "description": "Buttery, flaky croissant served with smooth, refreshing iced coffee. A classic combo that's light, crisp, and energizing. Perfect for a quick, satisfying bite..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/be3c8688-f975-4d1e-aad7-9229232bcc69_65c9a679-687d-4e66-9847-eb4d5c0f9825.png" + }, + { + "name": "Mcflurry Oreo ( S )", + "price": "104", + "description": "Delicious soft serve meets crumbled oreo cookies, a match made in dessert heaven. Perfect for one.. Contains: Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/a28369e386195be4071d9cf5078a438d" + }, + { + "name": "McFlurry Oreo ( M )", + "price": "129", + "description": "Delicious soft serve meets crumbled oreo cookies, a match made in dessert heaven. Share it, if you can.. Contains: Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f966500ed8b913a16cfdb25aab9244e4" + }, + { + "name": "Hot Fudge Sundae", + "price": "66", + "description": "A sinful delight, soft serve topped with delicious, gooey hot chocolate fudge. Always grab an extra spoon.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9c8958145495e8f2cf70470195f7834a" + }, + { + "name": "Strawberry Sundae", + "price": "66", + "description": "The cool vanilla soft serve ice cream with twirls of strawberry syrup.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/d7bd22aa47cffdcdde2d5b6223fde06e" + }, + { + "name": "Oreo Sundae ( M )", + "price": "72", + "description": "Enjoy the classic McFlurry Oreo goodness with a drizzle of hot fudge sauce with the Oreo Sundae!. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/3696da86802f534ba9ca68bd8be717ab" + }, + { + "name": "Black Forest McFlurry Medium", + "price": "139", + "description": "A sweet treat to suit your every mood. Contains: Soybeans, Peanut, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f513cc8c35cadd098835fb5b23c03561" + }, + { + "name": "Hot Fudge Brownie Sundae", + "price": "139", + "description": "Luscious chocolate brownie and hot-chocolate fudge to sweeten your day. Contains: Soybeans, Peanut, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6e00a57c6d8ceff6812a765c80e9ce74" + }, + { + "name": "Chocolate Overload McFlurry with Oreo Medium", + "price": "164.76", + "description": "Indulge in your chocolatey dreams with creamy soft serve, Oreo crumbs, a rich Hazelnut brownie, and two decadent chocolate sauces. Tempting, irresistible, and unforgettable. Contains: Gluten, Milk, Peanut, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/eb789769-9e60-4a84-9c64-90887ca79d7c_86154b6a-146c-47e9-9bbe-e685e4928e2e.png" + }, + { + "name": "Chocolate Overload McFlurry with Oreo Small", + "price": "134.28", + "description": "Indulge in your chocolatey dreams with creamy soft serve, Oreo crumbs, a rich Hazelnut brownie, and two decadent chocolate sauces. Tempting, irresistible, and unforgettable. Contains: Gluten, Milk, Peanut, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/431024a3-945d-4e6b-aafe-d35c410ac513_1b01e6dc-90cc-411d-a75e-9b98b42e178d.png" + } +] \ No newline at end of file diff --git a/firecrawl_menu.json b/firecrawl_menu.json new file mode 100644 index 00000000..ce78f263 --- /dev/null +++ b/firecrawl_menu.json @@ -0,0 +1,28 @@ +{ + "items": [ + { + "name": "Cart", + "url": "https://www.swiggy.com/checkout" + }, + { + "name": "Sign In", + "url": null + }, + { + "name": "Help", + "url": "https://www.swiggy.com/support" + }, + { + "name": "OffersNEW", + "url": "https://www.swiggy.com/offers-near-me" + }, + { + "name": "Search", + "url": "https://www.swiggy.com/search" + }, + { + "name": "Swiggy Corporate", + "url": "https://www.swiggy.com/corporate" + } + ] +} \ No newline at end of file diff --git a/menu.json b/menu.json new file mode 100644 index 00000000..810be887 --- /dev/null +++ b/menu.json @@ -0,0 +1,1688 @@ +[ + { + "name": "Big Yummy Cheese Burger", + "price": "349", + "description": "A spicy, cheesy indulgence, the Big Yummy Cheese Burger stacks a fiery paneer patty and a rich McCheese patty with crisp lettuce and smoky chipotle sauce on a Quarter Pound bun.. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/916ad567-194a-449a-a9f3-c53acc0fa52e_4dfe7bfa-a200-4eab-9ffe-532236399652.png" + }, + { + "name": "Big Yummy Cheese Meal (M).", + "price": "424.76", + "description": "Double the indulgence, double the flavor: our Big Yummy Cheese Burger meal layers a spicy paneer patty and Cheese patty with crisp lettuce and smoky chipotle sauce, served with fries (M) and a beverage of your choice.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/7c1c4952-781a-4fe7-ad31-d6650fccbbdd_19f5f0fa-2cb3-405c-818a-457c84ba5a01.png" + }, + { + "name": "Big Yummy Chicken Burger", + "price": "349", + "description": "Crafted for true indulgence, tender grilled chicken patty meets the McCrispy chicken patty, elevated with crisp lettuce, jalapenos, and bold chipotle sauce.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/8bdf918d-2571-4f8c-a8f6-bcae8a257144_dbbba789-1e27-4bd0-a356-d7f5ed17c103.png" + }, + { + "name": "Big Yummy Chicken Meal (M).", + "price": "424.76", + "description": "Indulge in double the delight: our Big Yummy Chicken Burger meal pairs the tender grilled chicken patty and Crispy chicken patty with crisp lettuce, jalapeños, and bold chipotle sauce, served with fries (M) and a beverage of your choice ..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/be8e14ab-7861-4cf7-9550-d4a58f09d28c_958f0a00-5a1d-4041-825c-e196ae06d524.png" + }, + { + "name": "Cappuccino (S) + Iced Coffee (S)", + "price": "199.04", + "description": "Get the best coffee combo curated just for you!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/19/5aeea709-728c-43a2-ab00-4e8c754cec74_494372be-b929-496e-8db8-34e128746eb9.png" + }, + { + "name": "Veg Pizza McPuff + McSpicy Chicken Burger", + "price": "260", + "description": "Tender and juicy chicken patty coated in spicy, crispy batter topped with a creamy sauce and crispy shredded lettuce will have you craving for more. Served with Veg Pizza McPuff.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/fb4b9d4775505e82d05d6734ef3e2491" + }, + { + "name": "2 Cappuccino", + "price": "233.33", + "description": "2 Cappuccino (S).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/34aaf6ee-06e2-4c60-9950-8ad569bc5898_2639ffd2-47dd-447b-bbbe-eb391315666f.png" + }, + { + "name": "McChicken Burger + McSpicy Chicken Burger", + "price": "315.23", + "description": "The ultimate chicken combo made just for you. Get the top selling McChicken with the McSpicy Chicken Burger..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/006d51070b0ab9c839a293b87412541c" + }, + { + "name": "2 Iced Coffee", + "price": "233.33", + "description": "Enjoy 2 Iced Coffee.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/e36a9ed2-bfc0-4d36-bfef-297e2c74f991_90e415d8-c4f0-45c1-ad81-db8ef52a5f96.png" + }, + { + "name": "McVeggie Burger + McAloo Tikki Burger", + "price": "210.47", + "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, makes our iconic McVeggie and combo with our top selling McAloo Tikki Burger..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/1f4d583548597d41086df0c723560da7" + }, + { + "name": "Strawberry Shake + Fries (M)", + "price": "196", + "description": "Can't decide what to eat? We've got you covered. Get this snacking combo with Medium Fries and Strawberry Shake..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/74603316fc90ea3cd2b193ab491fbf53" + }, + { + "name": "McChicken Burger + Fries (M)", + "price": "244.76", + "description": "Tender and juicy chicken patty cooked to perfection, with creamy mayonnaise and crunchy lettuce adding flavour to each bite. Served with Medium Fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/a321db80-a223-4a90-9087-054154d27189_9168f1ee-991b-4d8c-8e82-64b2ea249943.png" + }, + { + "name": "McVeggie Burger + Fries (M)", + "price": "215.23", + "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns. Served with Medium Fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/d14cc495747a172686ebe43e675bc941" + }, + { + "name": "McAloo Tikki Burger + Veg Pizza McPuff + Coke", + "price": "190.47", + "description": "The ultimate veg combo made just for you. Get the top selling McAloo Tikki served with Veg Pizza McPuff and Coke..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9f0269a2d28f4918a3b07f63a487f26d" + }, + { + "name": "McAloo Tikki + Fries (R)", + "price": "115.23", + "description": "Aloo Tikki+ Fries (R).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/1ffa9f16-d7ce-48d8-946a-aaac56548c88_10b13615-190f-4f06-91cb-ac7499046fb8.png" + }, + { + "name": "Mexican McAloo Tikki Burger + Fries (R)", + "price": "120", + "description": "A fusion of international taste combined with your favourite aloo tikki patty, layered with shredded onion, and delicious Chipotle sauce. Served with Regular Fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7274d82212a758e597550e8c246fb2f7" + }, + { + "name": "McChicken Burger + Veg Pizza McPuff", + "price": "184.76", + "description": "Tender and juicy chicken patty cooked to perfection, with creamy mayonnaise and crunchy lettuce adding flavour to each bite. Served with Veg Pizza McPuff..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9a66b8ef66d780b9f83a0fc7cd434ded" + }, + { + "name": "McVeggie Burger + Veg Pizza McPuff", + "price": "195.23", + "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns. Served with Veg Pizza McPuff..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/a34eb684-5878-4578-8617-b06e24e46fba_36b736e7-3a91-4618-bf0f-ce60d45e55d2.png" + }, + { + "name": "McVeggie Burger + Fries (R)", + "price": "184.76", + "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns. Served with Regular fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/5e29050f-38f9-4d42-9388-be895f2ba84b_591326c1-a753-44b9-b80c-462196c67fd0.png" + }, + { + "name": "Mexican McAloo Tikki Burger + Fries (L)", + "price": "180", + "description": "A fusion of international taste combined with your favourite aloo tikki patty, layered with shredded onion, and delicious Chipotle sauce. Served with Large Fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f4be6e877d2567a0b585d4b16e53871e" + }, + { + "name": "McChicken Burger + Fries (L)", + "price": "250.47", + "description": "Tender and juicy chicken patty cooked to perfection, with creamy mayonnaise and crunchy lettuce adding flavour to each bite. Served with Large Fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/08e794cb-6520-4907-890c-27449181c9fb_e30fa88b-c5aa-4b71-8c84-135dc433c954.png" + }, + { + "name": "McVeggie Burger + Fries (L)", + "price": "250.47", + "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns. Served with Large fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/b02d421c-fae4-4408-870d-3b51c4e4a94d_0a64d6d8-9eea-452f-917e-988d7db846e2.png" + }, + { + "name": "2 Fries (R)", + "price": "120", + "description": "World Famous Fries, crispy, golden, lightly salted and fried to perfection! Double your happiness with this fries combo.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/4a170da5ecae92e11410a8fbb44c8476" + }, + { + "name": "2 McVeggie Burger", + "price": "270.47", + "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns makes our iconic McVeggie..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/f1edf611-08aa-4b91-b99c-f135eb70df66_ce7bec39-1633-4222-89a2-40013b5d9281.png" + }, + { + "name": "Grilled Chicken & Cheese Burger + Coke", + "price": "239.99", + "description": "Flat 15% Off on Grilled Chicken & Cheese Burger + Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/5/29/18f3ce07-00b5-487b-8608-56440efff007_2b241e0a-fad5-4be4-a848-81c124c95b8b.png" + }, + { + "name": "McAloo Tikki Burger + Veg Pizza McPuff + Fries (R)", + "price": "208.57", + "description": "Flat 15% Off on McAloo Tikki + Veg Pizza McPuff + Fries (R).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/6d7aef29-bae6-403b-a245-38c3371a5363_5dc1ef9c-1f17-4409-946b-b2d78b8710d4.png" + }, + { + "name": "McVeggie Burger + Fries (M) + Piri Piri Mix", + "price": "240", + "description": "Flat 15% Off on McVeggie Burger + Fries (M) + Piri Piri Mix.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/3011bf9d-01fb-4ff3-a2ca-832844aa0dd0_8c231aef-ff56-4dd2-82e5-743a6240c37b.png" + }, + { + "name": "McVeggie Burger + Veg Pizza McPuff + Fries (L)", + "price": "290.47", + "description": "Flat 15% Off on McVeggie Burger + Veg Pizza McPuff + Fries (L).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/5/29/0e4d8ee8-ee6b-4e12-9386-658dbcdc6be4_6810365e-3735-4b21-860b-923a416403be.png" + }, + { + "name": "6 Pc Chicken Nuggets + Fries (M) + Piri Piri Spice Mix", + "price": "247.99", + "description": "The best Non veg sides combo curated for you! Get 6 pc Chicken McNuggets + Fries M. Top it up with Piri Piri mix..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/8cba0938b4401e5c8cb2ccf3741b93c4" + }, + { + "name": "McAloo Tikki Burger + Veg Pizza McPuff + Piri Piri Spice Mix", + "price": "128", + "description": "Get India's favourite burger - McAloo Tikki along with Veg Pizza McPuff and spice it up with a Piri Piri Mix.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/46bcb1e486cbe6dcdb0487b063af58a6" + }, + { + "name": "Grilled Chicken & Cheese Burger + Veg Pizza McPuff", + "price": "196", + "description": "A delicious Grilled Chicken & Cheese Burger + a crispy brown, delicious Pizza McPuff.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/80983b8d-5d94-45f7-84a8-57f2477933de_83c6e531-c828-4ecd-a68a-ed518677fb66.png" + }, + { + "name": "Corn & Cheese Burger + Veg Pizza McPuff", + "price": "184.76", + "description": "A delicious Corn & Cheese Burger + a crispy brown, delicious Pizza McPuff.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/6d92a078-e41e-4126-b70c-b57538675892_c28b61a2-2733-4b3c-a6a2-7943fa109e24.png" + }, + { + "name": "Corn & Cheese Burger + Fries (R)", + "price": "210.47", + "description": "A delicious Corn & Cheese Burger + a side of crispy, golden, world famous fries ??.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/c66d0139-6560-4be9-9239-762f9e80a31a_b4576f89-8186-461d-ba99-28a31a0507f9.png" + }, + { + "name": "Corn & Cheese Burger + Coke", + "price": "239.99", + "description": "Flat 15% Off on Corn & Cheese Burger + Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/5/31/043061fe-2b66-49a4-bc65-b26232584003_ec753203-f35a-487f-bc87-8dcccd31ea3f.png" + }, + { + "name": "Chocolate Flavoured Shake+ Fries (M)", + "price": "196", + "description": "Can't decide what to eat? We've got you covered. Get this snacking combo with Medium Fries and Chocolate Flavoured Shake..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/46781c13d587e5f951ac1bbb39e57154" + }, + { + "name": "2 McFlurry Oreo (S)", + "price": "158", + "description": "Delicious soft serve meets crumbled oreo cookies, a match made in dessert heaven. Make it double with this combo!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/65f3574f53112e9d263dfa924b1f8fed" + }, + { + "name": "2 Hot Fudge Sundae", + "price": "156", + "description": "A sinful delight, soft serve topped with delicious, gooey hot chocolate fudge. So good you won't be able to stop at one!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f1238db9da73d8ec7a999792f35865d9" + }, + { + "name": "McSpicy Chicken Burger + Fries (M) + Piri Piri Spice Mix", + "price": "295.23", + "description": "Tender and juicy chicken patty coated in spicy, crispy batter topped with a creamy sauce and crispy shredded lettuce will have you craving for more. Served with the spicy piri piri mix and medium fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/e10982204687e18ee6541684365039b8" + }, + { + "name": "McSpicy Paneer Burger + Fries (M) + Piri Piri Spice Mix", + "price": "295.23", + "description": "Rich and filling cottage cheese patty coated in spicy, crispy batter topped with a creamy sauce and crispy shredded lettuce will have you craving for more. Served with the spicy piri piri mix and medium fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/87ce9779f986dcb21ab1fcfe794938d1" + }, + { + "name": "McSpicy Paneer + Cheesy Fries", + "price": "295.23", + "description": "Rich and filling cottage cheese patty coated in spicy, crispy batter topped with a creamy sauce and crispy shredded lettuce will have you craving for more. Served with Cheesy Fries..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/fcc2fb1635f8e14c69b57126014f0bd5" + }, + { + "name": "Black Forest Mcflurry (M) BOGO", + "price": "139", + "description": "Get 2 Black Forest McFlurry for the price of one!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/15/0319a787-bf68-4cca-a81c-f57d9d993918_9b42d2bc-f1bd-47d4-a5a0-6eb9944ec5cd.png" + }, + { + "name": "New McSaver Chicken Surprise", + "price": "119", + "description": "Enjoy a delicious combo of the new Chicken Surprise Burger with a beverage, now in a delivery friendly reusable bottle.. Contains: Sulphite, Soybeans, Milk, Egg, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/b8c0d92e-86b0-4efa-81c1-66be9b3f846b_ddd80150-1aa4-465c-8453-7f7e7650c6c9.png" + }, + { + "name": "New McSaver Chicken Nuggets (4 Pc)", + "price": "119", + "description": "Enjoy New McSaver Chicken Nuggets (4 Pc).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7bf83367ed61708817caefbc79a3c9eb" + }, + { + "name": "New McSaver McAloo Tikki", + "price": "119", + "description": "Enjoy New McSaver McAloo Tikki.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/ab4c47366f0e51ac0071f705b0f2d93e" + }, + { + "name": "New McSaver Pizza McPuff", + "price": "119", + "description": "Enjoy New McSaver Pizza McPuff.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/45fa406e76418771de26c37e8863fbb3" + }, + { + "name": "Chicken Surprise Burger + McChicken Burger", + "price": "204.76", + "description": "Enjoy the newly launched Chicken Surprise Burger with the iconic McChicken Burger.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/4/f1d9557a-6f86-4eb2-abd5-569f2e865de2_9b71c67a-45f1-41c1-9e1c-b80a934768da.png" + }, + { + "name": "Chicken Surprise Burger + Fries (M)", + "price": "170.47", + "description": "Enjoy the newly launched Chicken Surprise Burger with the iconic Fries (M).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/4/ccceb88b-2015-47a3-857b-21a9641d87ed_ad91cfe4-3727-4f23-a34e-4d09c8330709.png" + }, + { + "name": "Crispy Veggie Burger + Cheesy Fries", + "price": "320", + "description": "Feel the crunch with our newly launched Crispy Veggie Burger with Cheesy Fries.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/d81e09c2-e3c9-4b1e-862f-5b760c429a00_fa2d522d-2987-49c7-b924-965ddd970c0e.png" + }, + { + "name": "Crispy Veggie Burger + McAloo Tikki", + "price": "255.23", + "description": "Feel the crunch with our newly launched Crispy Veggie Burger + McAloo Tikki.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/476ed2c4-89b2-41d2-9903-be339a6a07e5_ff0d9d12-ecbd-43b9-97c9-dd3e3c20a5cb.png" + }, + { + "name": "Crispy Veggie Burger + Piri Piri Fries (M)", + "price": "320", + "description": "Feel the crunch with our newly launched Crispy Veggie Burger with Piri Piri Fries (M).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/abda9650-7216-4a8d-87ce-05342438db59_b501b87f-e104-466e-8af6-d3a9947e76ac.png" + }, + { + "name": "Mc Crispy Chicken Burger + Piri Piri Fries (M)", + "price": "360", + "description": "Feel the crunch with our newly launched McCrispy Chicken Burger with Piri Piri Fries (M).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/02bce46b-37fb-4aaa-a035-4daef4fe5350_1292d3ee-7a34-45a2-adc5-1e5b14b4752a.png" + }, + { + "name": "Mc Crispy Chicken Burger + Cheesy Fries", + "price": "370.47", + "description": "Feel the crunch with our newly launched McCrispy Chicken Burger with Cheesy Fries.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/170e8dd8-d9c1-4b67-ab84-cec5dd4a9e32_6d80a13b-024b-454f-b680-d62c7252cd4d.png" + }, + { + "name": "Chicken Surprise Burger + Cold Coffee", + "price": "266.66", + "description": "Start of your morning energetic and satisfied with our new exciting combo of - Chicken Surprise + Cold Coffee (R).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/10/30/01f2c1c4-e80b-48f0-9323-8e75775e08e3_3dd290a6-ed00-405b-a654-e46ed2854c81.png" + }, + { + "name": "McAloo Tikki Burger + Cold Coffee", + "price": "251.42", + "description": "Start of your morning energetic and satisfied with our new exciting combo of - McAloo Tikki +Cold Coffee (R).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/10/30/78de1f5b-7fa2-4e00-b81d-c6d6392cd9ea_6f6eaf41-6844-4349-b37c-49cb7be685b3.png" + }, + { + "name": "Choco Crunch Cookie + McAloo Tikki Burger", + "price": "144.76", + "description": "A crunchy, chocolatey delight meets the iconic Aloo Tikki Burger,sweet and savory, the perfect duo for your snack-time cravings!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/9229f5fb-a89c-49fc-ac40-45e59fdf69dd_e17fe978-1668-4829-812d-d00a1b46e971.png" + }, + { + "name": "Choco Crunch Cookie + McVeggie Burger", + "price": "223.80", + "description": "A crispy Choco Crunch Cookie and a hearty McVeggie Burger,your perfect balance of sweet indulgence and savory delight in every bite!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/61529326-5b1d-4917-b74f-1b3de9d4e8ef_8dbfb34e-5dd7-4a85-b0dd-3f38be677ff9.png" + }, + { + "name": "Lemon Ice Tea + Choco Crunch Cookie", + "price": "237.14", + "description": "A refreshing Lemon Iced Tea paired with a crunchy Choco Crunch Cookie, sweet, zesty, and perfectly balanced for a delightful treat!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/1b46fafc-e69f-4084-8d73-b47f4291e45b_4b0bdb80-9cd2-4b85-a6b5-13fc7348a3a3.png" + }, + { + "name": "Veg Pizza McPuff + Choco Crunch Cookie", + "price": "139.04", + "description": "A perfect snack duo, savoury, Veg Pizza McPuff paired with a crunchy, chocolatey Choco Crunch Cookie for a delicious treat!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/da76bf5a-f86c-44f7-9587-90ef5bdbe973_cf99d6eb-6cef-4f03-bd64-4e9b94523817.png" + }, + { + "name": "Butter Croissant + Cappuccino..", + "price": "209", + "description": "Buttery croissant paired with a rich, frothy cappuccino.Warm, comforting, and perfectly balanced.A timeless duo for your anytime cravings..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/227a6aa3-f18c-4685-bb09-a76a01ec39a1_f347d737-b634-43d7-9b73-dc99bccde65f.png" + }, + { + "name": "Butter Croissant + Iced Coffee.", + "price": "209", + "description": "Buttery, flaky croissant served with smooth, refreshing iced coffee. A classic combo that's light, crisp, and energizing. Perfect for a quick, satisfying bite..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/be3c8688-f975-4d1e-aad7-9229232bcc69_65c9a679-687d-4e66-9847-eb4d5c0f9825.png" + }, + { + "name": "1 Pc Crispy Fried Chicken", + "price": "108", + "description": "Enjoy the incredibly crunchy and juicy and Crispy Fried Chicken- 1 Pc.. Contains: Egg, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/4e7f77ef46d856205d3e4e4913ffc0e9" + }, + { + "name": "2 Crispy Fried Chicken + 2 McSpicy Fried Chicken + 2 Dips + 2 Coke", + "price": "548.99", + "description": "A combo of crunchy, juicy fried chicken and spicy, juicy McSpicy chicken, with 2 Dips and chilled Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/305180a8-d236-4f9f-9d52-852757dfc0f6_ab67db7f-eb5d-4519-88f5-a704b666db4e.png" + }, + { + "name": "2 Pc Crispy Fried Chicken", + "price": "219", + "description": "Enjoy 2 Pcs of the incredibly crunchy and juicy and Crispy Fried Chicken.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/07ed0b2a381f0a21d07888cf1b1216eb" + }, + { + "name": "1 McSpicy Fried Chk + 1 Crispy Fried Chk + 4 Wings + 2 Coke + 2 Dips", + "price": "532.37", + "description": "Juicy and spicy McSpicy chicken, crispy fried chicken, and wings with 2 Dips and Coke perfect for a flavorful meal..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/14c7ed0d-5164-4411-8c52-2151e93801be_4bf5a209-caed-4b44-91b8-48cd71455f2e.png" + }, + { + "name": "1 Pc McSpicy Fried Chicken", + "price": "114", + "description": "Try the new McSpicy Fried chicken that is juicy, crunchy and spicy to the last bite!. Contains: Sulphite, Soybeans, Peanut, Milk, Egg, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/ceb4e0a0-9bff-48f8-a7da-7e8f53c38b86_c9005198-cc85-420a-81ec-7e0bca0ca8ab.png" + }, + { + "name": "2 Pc McSpicy Chicken Wings", + "price": "93", + "description": "Enjoy the 2 pcs of the New McSpicy Chicken Wings. Spicy and crunchy, perfect for your chicken cravings. Contains: Sulphite, Soybeans, Peanut, Milk, Egg, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6e4918a1cf1113361edba3ed33519ffc" + }, + { + "name": "2 Pc McSpicy Fried Chicken", + "price": "217", + "description": "Try the new McSpicy Fried chicken- 2 pcs that is juicy, crunchy and spicy to the last bite!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/057b4bdb-9b88-4c9a-b54a-fc41d0ea1b5f_c6cd0222-69ae-4f30-b588-9741b82d9195.png" + }, + { + "name": "3 Pc McSpicy Fried Chicken Bucket + 1 Coke", + "price": "380.99", + "description": "Share your love for chicken with 3 pcs of McSpicy Fried Chicken with refreshing coke. The perfect meal for your catchup!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/9832febf-e35a-4575-89c1-dfdefd42bb92_42385d00-210d-4554-ac0f-236268e404a3.png" + }, + { + "name": "4 Pc McSpicy Chicken Wings", + "price": "185", + "description": "Enjoy the 4 pcs of the New McSpicy Chicken Wings. Spicy and crunchy, perfect for your chicken cravings.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7122300975cc9640e84cdc7e7c74e042" + }, + { + "name": "4 Pc McSpicy Fried Chicken Bucket", + "price": "455", + "description": "Share your love for chicken with 4 pcs of McSpicy Fried Chicken..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/5cacf92d-f847-4670-acc2-381f984790d1_60f07bc4-f161-4603-b473-527828f8df08.png" + }, + { + "name": "5 Pc McSpicy Fried Chicken Bucket", + "price": "590", + "description": "Share your love for chicken with 5 pcs of McSpicy Fried Chicken that is spicy to the last bite.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/1e617f1f-2573-475f-a50e-8c9dd02dd451_6ccb66a3-728b-4abb-8a03-4f3eed32da18.png" + }, + { + "name": "12 Pc Feast Chicken Bucket", + "price": "808.56", + "description": "Enjoy 12 pc bucket of 4 Pc McSpicy Fried Chicken + 4 Pc Crispy Fried Chicken+ 4 pc McSpicy Chicken Wings + 2 Medium Cokes + 2 Dips. (Serves 3-4).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/a09f66d7-3ad3-4f8c-9168-6a95849430f5_e360b6f1-659f-42c0-abc0-7f05b95496ce.png" + }, + { + "name": "Chicken Lover's Bucket", + "price": "599.04", + "description": "Enjoy this crunchy combination of 4 Pc McSpicy Chicken Wings + 2 Pc McSpicy Fried Chicken + 2 Pc Crispy Fried Chicken+ 2 Dips + 2 Cokes. A chicken lover's dream come true! (Serves 3-4).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/650f5998-10b0-4c07-bb3b-e68036242f11_c9b5e73c-4bbf-4c04-b918-dd7a32b7973b.png" + }, + { + "name": "4 Chicken Wings + 2 McSpicy Fried Chicken + 2 Coke + 2 Dips", + "price": "528.56", + "description": "Spicy, juicy McSpicy Chicken wings and 2 Pc McSpicy Fried chicken with 2 Dips, paired with 2 chilled cokes.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/42ff8854-3701-4b8c-aace-da723977c2e3_8b62f3cc-32eb-4e84-949e-375045125efc.png" + }, + { + "name": "4 McSpicy Fried Chicken Bucket + 2 Dips + 2 Coke", + "price": "532.37", + "description": "4 pieces of juicy, spicy McSpicy Fried Chicken with 2 Dips and the ultimate refreshment of chilled coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/305180a8-d236-4f9f-9d52-852757dfc0f6_ab67db7f-eb5d-4519-88f5-a704b666db4e.png" + }, + { + "name": "5 McSpicy Fried Chicken Bucket + 2 Dips + 2 Coke", + "price": "566.66", + "description": "5 pieces of juicy, spicy McSpicy Fried chicken with 2 Dips and 2 refreshing Cokes..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/29742264-8fee-4b8f-a8e2-52efb5d4edf9_314d6cbc-b4ea-4d92-92ea-5b7d1df70a9a.png" + }, + { + "name": "8 McSpicy Chicken Wings Bucket + 2 Coke + 1 Dip", + "price": "465.71", + "description": "Juicy, spicy McSpicy Chicken wings with 1 Dip and the ultimate refreshment of chilled coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/ea6a4090-5c1d-4b32-926b-5e1de33574ba_242accc3-d883-4434-9913-fb6c35574c9b.png" + }, + { + "name": "Chicken Surprise Burger with Multi-Millet Bun", + "price": "84.15", + "description": "Try the Chicken Surprise Burger in the new multi-millet bun! Enjoy the same tasty chicken patty you love, now sandwiched between a nutritious multi-millet bun.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/1fb900f6-71bb-4724-a507-830735f555c5_06c06358-bc36-469d-84e3-44e9504bb7d0.png" + }, + { + "name": "McAloo Tikki Burger with Multi-Millet Bun", + "price": "83.91", + "description": "Try your favourite McAloo Tikki Burger in a multi-millet bun! Enjoy the same tasty McAloo Tikki patty you love, now sandwiched between a nutritious millet bun..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/cfd7d779-b04d-43be-947e-a43df33ae119_a0de6b13-c9ac-489a-8416-e9b72ace4542.png" + }, + { + "name": "McChicken Burger with Multi-Millet Bun", + "price": "150.47", + "description": "Make a healthier choice with our McChicken Burger in a multi-millet bun! Same juicy chicken patty, now with a nutritious twist..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/bf0a9899-19a5-4fc9-9898-9e15098bac43_681fbed4-5354-4d5a-9d67-b59d274ea633.png" + }, + { + "name": "McSpicy Chicken Burger with Multi-Millet Bun", + "price": "221.76", + "description": "Feel the heat and feel good too! Try your McSpicy Chicken Burger in nutritious multi-millet bun..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/6eac1f61-cf7b-422a-bc6d-8c55bc1ff842_01e557a3-f5a2-4239-b06b-01ffd46ec154.png" + }, + { + "name": "McSpicy Paneer Burger with Multi-Millet Bun", + "price": "221.76", + "description": "Spice up your meal with a healthier bite! Try your McSpicy Paneer Burger with the nutritious multi-millet bun..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/70eb5956-3666-47f4-bcf8-1f5075207222_fd9e2a06-3f27-47e3-88aa-86895ffaa65e.png" + }, + { + "name": "McVeggie Burger with Multi-Millet Bun", + "price": "158.40", + "description": "Try your favorite McVeggie Burger in a nutritious multi-millet bun! A healthier twist on a classic favorite, with the same tasty veggie patty you love.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/397caee8-4da3-4143-98b1-e4dac155ab3e_0ed975df-7665-4f3a-ae16-8a7eb008afd6.png" + }, + { + "name": "Crispy Veggie Burger Protein Plus (1 Slice)", + "price": "226.71", + "description": "A flavourful patty made with a blend of 7 Premium veggies topped with zesty cocktail sauce now a protein slice to fuel you up, all served between soft premium buns. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/2fcff334-211c-4a2b-bee8-018ce9f10572_71ad5b2a-982d-407a-a833-8f3c1c6bf240.png" + }, + { + "name": "Crispy Veggie Burger Protein Plus (2 Slices)", + "price": "253.43", + "description": "A flavourful patty made with a blend of 7 Premium veggies topped with zesty cocktail sauce now a protein slice to fuel you up, all served between soft premium buns. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/e2c0ecf8-1ca0-447c-a56f-1b8758f7a406_314e4442-4db5-403e-b71c-ffa797622eee.png" + }, + { + "name": "Crispy Veggie Burger Protein Plus + Corn + Coke Zero", + "price": "399", + "description": "A flavourful patty made with a blend of 7 premium veggies, topped with zesty cocktail sauce and now a protein slice to fuel you up, all served between soft premium buns. Paired with corn and Coke Zero..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/d0ac58e8-e601-49e1-abf2-a5456a63f057_a8bb7584-906a-4ce1-891b-6e877f6b4543.png" + }, + { + "name": "McEgg Burger Protein Plus (1 Slice)", + "price": "89.10", + "description": "A steamed egg , spicy habanero sauce and onions and a tasty new protein slice. Simple, satisfying and powered with protein.. Contains: Gluten, Milk, Egg, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/0482392e-9b78-466d-93db-e653c77e0e35_55c40d44-9050-47eb-902e-5639c1423ee7.png" + }, + { + "name": "McEgg Burger Protein Plus (2 Slices)", + "price": "117.80", + "description": "A steamed egg , spicy habanero sauce and onions and a tasty new protein slice. Simple, satisfying and powered with protein.. Contains: Gluten, Milk, Egg, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/a5750dc8-8a1f-48a0-8fa2-83007ee4377c_c8501c16-90f4-4943-80f7-719f519fb918.png" + }, + { + "name": "McAloo Tikki Burger Protein Plus (1 Slice)", + "price": "89.10", + "description": "The OG Burger just got an upgrade with a tasty protein slice.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/6c4f7375-3424-4d01-8425-509381fb3015_25ebe6f6-2f23-4b3f-8dcb-02ea740f91e9.png" + }, + { + "name": "McAloo Tikki Burger Protein Plus (2 Slices)", + "price": "117.80", + "description": "The OG Burger just got an upgrade with a tasty protein slice.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/a95ad198-b6d8-43b0-9960-a1b8d8ceb6f2_12b5df65-ba9d-41ff-b762-88e391b121f8.png" + }, + { + "name": "McAloo Tikki Burger Protein Plus+ Corn + Coke Zero", + "price": "299", + "description": "The OG McAloo Tikki Burger just got an upgrade with a tasty protein slice. Served with buttery corn and a refreshing Coke Zero for a nostalgic yet balanced combo..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/da8beade-c083-4396-b4ff-289b647af81d_b0f6b946-7151-45e3-9fe0-e15a2c46c1d6.png" + }, + { + "name": "McCheese Chicken Burger Protein Plus (1 Slice)", + "price": "297", + "description": "Double the indulgence with sinfully oozing cheesey patty and flame grilled chicken patty , along with chipotle sauce , shredded onion , jalapenos , lettuce and now with a protein slice. Indulgent meets protein power.. Contains: Gluten, Egg, Milk, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/d32ea9af-d134-4ae9-8a1a-8dafddd53cb7_da16f662-ce98-4990-ab8c-566697743730.png" + }, + { + "name": "McCheese Chicken Burger Protein Plus (2 Slices)", + "price": "324.72", + "description": "Double the indulgence with sinfully oozing cheesey patty and flame grilled chicken patty , along ith chipotle sauce , shredded onion , jalapenos , lettuce and now with a protein slice. Indulgent meets protein power.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/e2b28b79-63df-43bb-8c41-240a076e0a3a_49dbc6df-3f9d-448c-bd94-377ebcc3a335.png" + }, + { + "name": "McCheese Chicken Burger Protein Plus + 4 Pc Chicken Nuggets+ Coke Zero", + "price": "449", + "description": "Double the indulgence with sinfully oozing cheesy patty and flame-grilled chicken patty, chipotle sauce, shredded onion, jalapenos, lettuce, and now a protein slice. Served with 4 Pc Chicken nuggets and Coke Zero. Indulgent meets protein power..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/f9aae1ec-6313-4684-aeb3-50de503fcf23_f5b200ed-60fa-42f6-928f-6b95327da3bb.png" + }, + { + "name": "McChicken Burger Protein Plus (1 Slice)", + "price": "185.13", + "description": "The classic McChicken you love, made more wholesome with a protein slice. Soft, savoury, and now protein rich.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/20fd3d59-0003-4e80-ad71-79ca8ae4d050_8bf1219b-a67d-4518-a115-167fa10bc440.png" + }, + { + "name": "McChicken Burger Protein Plus + 4 Pc Chicken Nuggets + Coke Zero", + "price": "349", + "description": "The classic McChicken you love, made more wholesome with a protein slice. Soft, savoury, and now protein-rich. Comes with 4 crispy Chicken nuggets and a chilled Coke Zero..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/afdcdd59-c5f0-44f1-9014-b28455ce0ecf_059ea761-4f24-4e04-b152-cdb82a04b366.png" + }, + { + "name": "McChicken Protein Burger Plus (2 Slices)", + "price": "212.84", + "description": "The classic McChicken you love, made more wholesome with a protein slice.Soft, savoury, and now protein-rich.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/1e1f281e-8378-4993-a7c1-d6f319a7bd17_768a7827-1fe8-43a6-ba35-a0ccb5a95ef5.png" + }, + { + "name": "McCrispy Chicken Burger Protein Plus (1 Slice)", + "price": "246.51", + "description": "A Crunchy , golden chicken thigh fillet , topped with fresh lettuce and creamy pepper mayo now also with a hearty protein slice all nestled between soft toasted premium buns.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/4f908fa6-2fa5-4367-9da9-9039cb5e72e0_c8570551-57ae-441a-81c9-64335392ac3b.png" + }, + { + "name": "McCrispy Chicken Burger Protein Plus (2 Slices)", + "price": "276.20", + "description": "A Crunchy , golden chicken thigh fillet , topped with fresh lettuce and creamy pepper mayo now also with a hearty protein slice all nestled between soft toasted premium buns.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/fc4e713d-5ee0-421e-9690-7c8b2c587e5a_2b710d54-3ae7-46fe-990c-140b34494dee.png" + }, + { + "name": "McCrispy Chicken Burger Protein Plus + 4 Pc Chicken Nugget + Coke Zero", + "price": "419", + "description": "A crunchy, golden chicken thigh fillet topped with fresh lettuce and creamy pepper mayo, now also with a hearty protein slice, all nestled between soft toasted premium buns. Comes with 4-piece nuggets and Coke Zero..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/fdb3d9b1-2262-4b18-9264-9bd00b3213ba_20858442-9b85-4728-b7b2-07d72349782e.png" + }, + { + "name": "McEgg Burger Protein Plus + 4 Pc Chicken Nuggets + Coke Zero", + "price": "299", + "description": "A steamed egg, spicy habanero sauce, onions, and a tasty new protein slice. Simple, satisfying, and powered with protein. Served with 4-piece chicken nuggets and Coke Zero..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/47092b74-fac3-48a2-9abc-1d43b975199f_d07e626a-6cb9-4664-8517-4ce7ba71b234.png" + }, + { + "name": "McSpicy Chicken Burger Protein Plus (1 Slice)", + "price": "225.72", + "description": "Indulge in our signature tender chicken patty coated in spicy crispy batter , topped with creamy sauce ,crispy lettuce and now with a new protein slice .. Contains: Gluten, Milk, Egg, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/79035335-855d-4b89-93e4-c063258aaf64_22e9ada8-6768-481d-83d3-7223bc119bce.png" + }, + { + "name": "McSpicy Chicken Burger Protein Plus (2 Slices)", + "price": "252.44", + "description": "Indulge in our signature tender chicken patty coated in spicy crispy batter , topped with creamy sauce ,crispy lettuce and now with a new protein slice .. Contains: Gluten, Egg, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/36e2e35c-86d5-4b0d-b573-325c7b4e5fd9_4bc54386-d43e-4d28-baca-d0da22bc3668.png" + }, + { + "name": "McSpicy Chicken Burger Protein Plus + 4 Pc Chicken Nuggets + Coke Zero", + "price": "399", + "description": "Indulge in our signature tender chicken patty coated in spicy crispy batter , topped with creamy sauce ,crispy lettuce and now with a new protein slice Served with 4-piece chicken nuggets and Coke Zero..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/a81bd599-05ad-4453-9ada-499d9cf45b29_1096d08c-d519-4133-babb-2f54a9b6e5f0.png" + }, + { + "name": "McSpicy Paneer Burger Protein Plus (1 Slice)", + "price": "226.71", + "description": "Indulge in rich and filling spicy paneer patty served with creamy sauce, crispy lettuce and now with a new protein slice. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/3ca8296e-9885-4df6-8621-5349298a150f_a2ad6b56-b6ae-4e4b-986c-abe75e052d75.png" + }, + { + "name": "McSpicy Paneer Burger Protein Plus (2 Slices)", + "price": "253.43", + "description": "Indulge in rich and filling spicy paneer patty served with creamy sauce, crispy lettuce and now with a new protein slice. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/c2561720-1064-4043-a143-3c9ae893e481_b09ff597-1b69-4098-ba04-6ab3f3b09c6f.png" + }, + { + "name": "McSpicy Paneer Burger Protein Plus + Corn + Coke Zero", + "price": "399", + "description": "Indulge in rich and filling spicy paneer patty served with creamy sauce, crispy lettuce and now with a new protein slice. Served with Corn and Coke Zero..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/d5c3f802-ad34-4086-ae66-731406a82c99_4b87555b-5e27-40da-b7f1-fd557212e344.png" + }, + { + "name": "McSpicy Premium Burger Veg Protein Plus (2 Slices)", + "price": "303.93", + "description": "A wholesome spicy paneer patty, lettuce topped with jalapenos and cheese slice and now with a protein-packed slice for that extra boost , spicy cocktail sauce and cheese sauce.. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/05949fc3-08a4-4e47-8fd1-16abc12a952c_0b80fed2-213d-4945-9ea7-38c818cdb6ef.png" + }, + { + "name": "McSpicy Premium Chicken Burger Protein Plus (1 Slice)", + "price": "287.10", + "description": "A wholesome spicy chicken patty lettuce topped with jalapenos and cheese slice plus an added protein slice , spicy cocktail sauce and cheese sauce.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/690ecde3-4611-4323-8436-a1dadfe2eb7b_fe9b5993-f422-4bae-8211-9715dd1d51d8.png" + }, + { + "name": "McSpicy Premium Chicken Burger Protein Plus (2 Slices)", + "price": "315.80", + "description": "A wholesome spicy chicken patty lettuce topped with jalapenos and cheese slice plus an added protein slice , spicy cocktail sauce and cheese sauce.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/80bf28f3-b317-4846-b678-95cfc857331a_d247479e-0bcb-4f5d-90da-b214d30e70a3.png" + }, + { + "name": "McSpicy Premium Chicken Protein Plus + 4 Pc Chicken Nugget + Coke Zero", + "price": "479", + "description": "A wholesome spicy chicken patty, lettuce topped with jalapenos and cheese slice, plus an added protein slice. Comes with spicy cocktail sauce and cheese sauce,served with 4 Pc crispy chicken nuggets and a Coke Zero..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/a512848a-d34d-44cf-a0db-f7e84781c63d_6d8fa5df-a1c6-42e6-b88b-929c59c4f50c.png" + }, + { + "name": "McSpicy Premium Veg Burger Protein Plus (1 Slice)", + "price": "276.20", + "description": "A wholesome spicy paneer patty, lettuce topped with jalapenos and cheese slice and now with a protein packed slice for that extra boost , spicy cocktail sauce and cheese sauce.. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/8768ff6a-ea10-4260-91e6-018695820d79_2399dc04-b793-4336-b2e4-a6240cdc5f78.png" + }, + { + "name": "McSpicy Premium Veg Burger Protein Plus + Corn + Coke Zero", + "price": "449", + "description": "A wholesome spicy paneer patty, lettuce topped with jalapenos and cheese slice, now with a protein-packed slice for that extra boost. Comes with spicy cocktail sauce, cheese sauce, buttery corn, and a chilled Coke Zero..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/d183fbeb-62dc-4b3d-936f-99ddc9dc5466_ce66dc12-da9f-479d-bd4a-c77fdb3dae2c.png" + }, + { + "name": "McVeggie Burger Protein Plus (1 Slice)", + "price": "185.13", + "description": "The classic McVeggie you love, made more wholesome with a protein slice.Soft, savoury, and now protein-rich.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/99aedaa8-836b-4893-b7f9-24b3303b6c94_781b8449-5bc9-42e5-9747-00bcec474deb.png" + }, + { + "name": "McVeggie Burger Protein Plus (2 Slices)", + "price": "212.88", + "description": "The classic McVeggie you love, made more wholesome with a protein slice.Soft, savoury, and now protein rich.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/630f74dd-841a-4641-bf7f-c23cd9e4e0d5_0fd861eb-43f6-46c1-9bfe-f8043c2aae6a.png" + }, + { + "name": "McVeggie Burger Protein Plus + Corn + Coke Zero", + "price": "339", + "description": "The classic McVeggie you love, made more wholesome with a protein slice. Soft, savoury, and now protein rich. Served with sweet corn and Coke Zero for a balanced meal..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/f448d30d-9bbe-4659-bbb4-14b22d93b0ae_0a188512-5619-4d62-980f-7db8e45f6ebe.png" + }, + { + "name": "Big Group Party Combo 6 Veg", + "price": "760.95", + "description": "Enjoy a Big Group Party Combo of McAloo + McVeggie + McSpicy Paneer + Mexican McAloo + Corn and Cheese + Crispy Veggie burger.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/427a977d-1e14-4fe9-a7ac-09a71f578514_e3089568-134d-45fe-b97e-59d756892f15.png" + }, + { + "name": "Big Group Party Combo for 6 Non- Veg", + "price": "856.19", + "description": "Enjoy a Big Group Party Combo of Surprise Chicken + McChicken + McSpicy Chicken + Grilled Chicken + McSpicy Premium + Mc Crispy Chicken Burger .", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/e1a453be-6ae4-4e82-8957-c43756d2d72a_d7cac204-1cb6-4b6d-addf-c0324435cc58.png" + }, + { + "name": "Big Group Party Combo1 for 4 Non- Veg", + "price": "475.23", + "description": "Save on your favourite Big Group Party Combo - Surprise Chicken + McChicken + McSpicy Chicken + Grilled Chicken Burger.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/d955dfa6-3a2c-46d2-b59c-e9e59b0939c3_1417a17c-bb7e-4e24-8931-3950cac2f074.png" + }, + { + "name": "Big Group Party Combo1 for 4 Veg", + "price": "475.23", + "description": "Get the best value in your Combo for 4 Save big on your favourite Big Group Party Combo-McAloo + McVeggie + McSpicy Paneer + Corn and Cheese Burger.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/5c723cbf-e7e2-48cc-a8fe-6f4e0eeea73d_899a4709-e9fb-4d9f-a997-973027fc0e7d.png" + }, + { + "name": "Big Group Party Combo2 for 4 Non-Veg", + "price": "522.85", + "description": "Your favorite party combo of 2 McChicken + 2 McSpicy Chicken Burger.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/b648ba7b-e82d-4d37-ab47-736fdb12cd90_19ef381a-3cc6-4b50-9e69-911f3656987f.png" + }, + { + "name": "2 Crispy Veggie Burger + Fries (L) + 2 Coke", + "price": "635.23", + "description": "Feel the crunch with Burger Combos for 2: 2 Crispy Veggie Burger + Fries (L)+ 2 Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/ce6551d8-829a-4dde-9131-cbf7818a6f26_a5b69cee-6377-4295-bbee-49725693c45d.png" + }, + { + "name": "Big Group Party Combo2 for 4 Veg", + "price": "522.85", + "description": "Your favorite party combo of 2 McVeggie + 2 McSpicy Paneer Burger.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/245c7235-500d-4212-84bd-f60f9aaf27be_73b24a53-d304-4db0-9336-061760ca17a9.png" + }, + { + "name": "2 Crispy Veggie Burger + 2 Fries (M) + Veg Pizza McPuff", + "price": "604.76", + "description": "Feel the crunch with Burger Combos for 2: 2 Crispy Veggie Burger + 2 Fries (M)+ Veg Pizza McPuff.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/ca2023e7-41e0-4233-a92c-c2bc52ef8ee5_33a0ea24-50ee-4a9b-b917-bbc2e6b5f37b.png" + }, + { + "name": "Crispy Veggie Burger + McVeggie Burger + Fries (M)", + "price": "424.76", + "description": "Feel the crunch with Crispy Veggie Burger+ McVeggie + Fries (M).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/c6dd67c1-0af9-4333-a4b5-3501e0ce4579_2fee69c9-f349-46c5-bdd9-8965b9f76687.png" + }, + { + "name": "Burger Combo for 2: McAloo Tikki", + "price": "364.76", + "description": "Stay home, stay safe and share a combo- 2 McAloo Tikki Burgers + 2 Fries (L).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/ea7ba594c7d77cb752de9a730fbcb3bf" + }, + { + "name": "6 Pc Chicken Nuggets + McChicken Burger + Coke", + "price": "375.22", + "description": "Tender and juicy chicken patty cooked to perfection, with creamy mayonnaise and crunchy lettuce adding flavour to each bite. Served with 6 Pc Nuggets and Coke..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/934194567f9c231dc46dccf2d4e6d415" + }, + { + "name": "Burger Combo for 2: McSpicy Chicken + McChicken", + "price": "464.76", + "description": "Flat 15% Off on McSpicy Chicken Burger + McChicken Burger + Fries (M).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/10ada13e-5724-487f-8ab6-fd07005859ad_a57d565d-0dfe-4424-bc5b-77b16143ad63.png" + }, + { + "name": "Burger Combo for 2: Corn & Cheese + McVeggie", + "price": "404.76", + "description": "Flat 15% Off on Corn & Cheese Burger +McVeggie Burger+Fries (M).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/08e9bc73-6774-41cf-96bb-ca817c4e23d3_e00ec89e-86f8-4531-956a-646082dc294c.png" + }, + { + "name": "Burger Combo for 2: McSpicy Chicken Burger with Pizza McPuff", + "price": "535.23", + "description": "Save big on your favourite sharing combo- 2 McSpicy Chicken Burger + 2 Fries (M) + Veg Pizza McPuff.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/e62aea3ba1cd5585a76004f59cd991e5" + }, + { + "name": "Burger Combo for 2: McSpicy Paneer + McAloo Tikki with Pizza McPuff", + "price": "427.61", + "description": "Get the best value in your meal for 2. Save big on your favourite sharing meal - McSpicy Paneer Burger + 2 Fries (M) + McAloo Tikki Burger + Veg Pizza McPuff.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/0ea8a2fddbbc17bc6239a9104963a3e8" + }, + { + "name": "Burger Combo for 2: McChicken Burger", + "price": "464.75", + "description": "Save big on your favourite sharing combo - 2 McChicken Burger + Fries (L) + 2 Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/7/177036e5-4afe-4076-b9cd-d7031f60ffe8_97687ef4-e242-4625-9b3c-398c60b8ddf2.png" + }, + { + "name": "Burger Combo for 2: McSpicy Chicken Burger", + "price": "548.56", + "description": "Save big on your favourite sharing combo - 2 McSpicy Chicken Burger + Fries (L) + 2 Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/98afaf26d81b15bec74cc356fe60cc13" + }, + { + "name": "Burger Combo for 2: McVeggie Burger", + "price": "424.75", + "description": "Save big on your favourite sharing combo - 2 McVeggie Burger + Fries (L) + 2 Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/09b3cb6130cfae15d486223c313fb6c6" + }, + { + "name": "2 Chicken Maharaja Mac Burger + 2 Coke + Fries (L) + McFlurry Oreo (M)", + "price": "670.47", + "description": "Enjoy 2 of the tallest burgers innovated by us. Created with chunky juicy grilled chicken patty paired along with fresh ingredients like jalapeno, onion, slice of cheese, tomatoes & crunchy lettuce dressed with the classical Habanero sauce. Served with Coke, Large Fries and a medium McFlurry Oreo.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/65c9c9b82c4d1f77a05dc4d89c9ead1d" + }, + { + "name": "Burger Combo for 2: Corn & Cheese Burger", + "price": "464.75", + "description": "Save big on your favourite sharing combo - 2 Corn and Cheese Burger + Fries (L) + 2 Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/847b562672e71c2352d92b797c0b0a4e" + }, + { + "name": "Burger Combo for 2: Grilled Chicken & Cheese", + "price": "495.23", + "description": "Save big on your favourite sharing combo - 2 Grilled Chicken and Cheese Burger + Fries (L) + 2 Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/1166a8baa3066342affb829ef0c428dd" + }, + { + "name": "2 Mc Crispy Chicken Burger + Fries (L) + 2 Coke", + "price": "724.75", + "description": "Feel the crunch with our Burger Combos for 2 : 2 McCrispy Chicken Burger + Fries (L)+ 2 Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/b4fda3b0-82c9-4d97-ab9c-c804b7d7893a_a14992d5-49cb-4035-827d-a43e40752840.png" + }, + { + "name": "Mc Crispy Chicken Burger + McChicken Burger + Fries (M)", + "price": "430.47", + "description": "Feel the crunch with McCrispy Chicken Burger+ McChicken + Fries (M).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/eaeeaf30-7bb6-4bef-9b78-643533a4520c_e54bb308-0447-44c2-9aaf-f36ce25f7939.png" + }, + { + "name": "Mc Crispy Chicken Burger + McSpicy Chicken Wings - 2 pc + Coke (M)", + "price": "415.23", + "description": "Feel the crunch with McCrispy Chicken Burger+ McSpicy Chicken Wings - 2 pc + Coke (M).", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/f8669731-b3e5-434d-8b45-269b75555b9b_3dfd8e01-5472-40db-a266-bb055036531b.png" + }, + { + "name": "McChicken Double Patty Burger Combo", + "price": "352.99", + "description": "Your favorite McChicken Burger double pattu burger + Fries (M) + Drink of your choice in a new, delivery friendly, resuable bottle..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/e8a8f43ee29a3b97d2fac37e89648eac" + }, + { + "name": "McSpicy Chicken Double Patty Burger combo", + "price": "418.99", + "description": "Your favorite McSpicy Chicken double patty Burger + Fries (M) + Drink of your choice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/91ed96b67df6e630a6830fc2e857b5b1" + }, + { + "name": "McVeggie Burger Happy Meal", + "price": "298.42", + "description": "Enjoy a combo of McVeggie Burger + Sweet Corn + B Natural Mixed Fruit Beverage + Book.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/15e7fc6b-f645-4ed9-a391-1e1edc452f9f_47ad6460-6af4-43b4-8365-35bb0b3fc078.png" + }, + { + "name": "McChicken Burger Happy Meal", + "price": "321.42", + "description": "Enjoy a combo of McChicken Burger + Sweet Corn + B Natural Mixed Fruit Beverage + Book.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/363ca5bf-bb04-4847-bc4e-5330a27d3874_8b5e46ed-61ba-4e8f-b291-2955af07eba0.png" + }, + { + "name": "McAloo Tikki Burger Happy Meal", + "price": "205.42", + "description": "Enjoy a combo of McAloo Tikki Burger + Sweet Corn + B Natural Mixed Fruit Beverage + Book.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/a9f2acb6-3fd3-4691-beb0-61439337f907_6733a4ee-3eb5-46da-9ab7-1b6943f68be7.png" + }, + { + "name": "Big Spicy Paneer Wrap Combo", + "price": "360.99", + "description": "Your favorite Big Spicy Paneer Wrap + Fries (M) + Drink of your choice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/22/116148a2-7f56-44af-bdd0-6bbb46f48baa_355bbd20-86d4-49b8-b3a9-d290772a9676.png" + }, + { + "name": "9 Pc Chicken Nuggets Combo", + "price": "388.98", + "description": "Enjoy your favorite Chicken McNuggets + Fries (M) + Drink of your choice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/4c56f086e500afe6b2025f9c46846e12" + }, + { + "name": "Mexican McAloo Tikki Burger Combo", + "price": "223.99", + "description": "Enjoy a delicious combo of Mexican McAloo Tikki Burger + Fries (M) + Beverage of your choice in a new, delivery friendly, reusable bottle..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/0140c49c7274cdb6af08053af1e6cc20" + }, + { + "name": "McEgg Burger Combo", + "price": "230.99", + "description": "Enjoy a combo of McEgg + Fries (M) + Drink of your Choice . Order now to experience a customizable, delicious meal..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/4f474c833930fa31d08ad2feed3414d8" + }, + { + "name": "McChicken Burger Combo", + "price": "314.99", + "description": "Your favorite McChicken Burger + Fries (M) + Drink of your choice in a new, delivery friendly, resuable bottle..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/b943fe56-212d-4366-a085-4e24d3532b30_3776df33-e18d-46c9-a566-c697897f1d16.png" + }, + { + "name": "McSpicy Chicken Burger Combo", + "price": "363.99", + "description": "Your favorite McSpicy Chicken Burger + Fries (M) + Drink of your choice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/214189a0-fd53-4f83-9d18-c3e53f2c017a_70748bf8-e587-47d6-886c-9b7ac57e84b3.png" + }, + { + "name": "McSpicy Paneer Burger Combo", + "price": "344.99", + "description": "Enjoy your favourite McSpicy Paneer Burger + Fries (M) + Drink of your Choice . Order now to experience a customizable, delicious combo.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/70fc7aa0-7f3b-418e-8ccd-5947b5f1aacd_055bbe77-fbe7-4200-84d3-4349475eb298.png" + }, + { + "name": "Veg Maharaja Mac Burger Combo", + "price": "398.99", + "description": "Enjoy a double decker Veg Maharaja Mac+ Fries (M) + Drink of your Choice . Order now to experience a customizable, delicious meal..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/d064fb5e-fb2e-4e1d-9515-18f26489f5b1_f6f49dba-2ae7-4ebf-8777-548c5ad4d799.png" + }, + { + "name": "McVeggie Burger Combo", + "price": "308.99", + "description": "Enjoy a combo of McVeggie + Fries (M) + Drink of your Choice in a new, delivery friendly, resuable bottle..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/5c9942c5-bc9d-4637-a312-cf51fe1d7aa8_e7e13473-9424-4eb8-87c3-368cfd084a2f.png" + }, + { + "name": "Big Spicy Chicken Wrap Combo", + "price": "379.99", + "description": "Your favorite Big Spicy Chicken Wrap + Fries (M) + Drink of your choice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/22/f631f834-aec2-410d-ab9d-7636cd53d7a0_8801a3c7-ad83-4f7c-bcb0-76101fadde91.png" + }, + { + "name": "Chicken Maharaja Mac Burger Combo", + "price": "398.99", + "description": "Enjoy a double decker Chicken Maharaja Mac + Fries (M) + Drink of your Choice . Order now to experience a customizable, delicious meal..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/ead090f2-5f80-4159-a335-d32658bcfc7c_8bcc5cd9-b22a-4f5d-8cde-0a2372c985c8.png" + }, + { + "name": "Grilled Cheese and Chicken Burger Combo", + "price": "310.47", + "description": "Enjoy a combo of Grilled Chicken & Cheese Burger + Fries (M) + Coke . Order now to experience a customizable, delicious meal..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/f6813404-54a4-492b-a03a-cf32d00ee1ae_c98c4761-69eb-4503-bde3-dffaafd43b15.png" + }, + { + "name": "Corn & cheese Burger Combo", + "price": "310.47", + "description": "Enjoy a combo of Corn & Cheese Burger + Fries (M) + Coke . Order now to experience a customizable, delicious meal..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/c8839f22-e525-44bb-8a50-8ee7cffecd26_cf4c0bcf-c1b9-4c0e-9109-03eb86abf4dd.png" + }, + { + "name": "Birthday Party Package - McChicken", + "price": "2169.14", + "description": "5 McChicken Burger + 5 Sweet Corn + 5 B Natural Mixed Fruit Beverage + 5 Soft Serve (M) + Book.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/15/37e0bc09-3e46-41d6-8146-66d6962ae2ab_0c91e294-cad2-4177-a2a9-86d6523bb811.png" + }, + { + "name": "Birthday Party Package - McVeggie", + "price": "2169.14", + "description": "5 McVeggie Burger + 5 Sweet Corn + 5 B Natural Mixed Fruit Beverage + 5 Soft Serve (M) + Book.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/15/0be6390f-e023-47f8-8bf4-9bb16d4995ce_2eceb8c2-6e90-47d7-97d6-93960391e668.png" + }, + { + "name": "McEgg Burger Happy Meal", + "price": "231.42", + "description": "Enjoy a combo of McEgg Burger + Sweet Corn + B Natural Mixed Fruit Beverage + Book.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/ada14e04-680d-44ef-bb05-180d0cc26ebc_b7bda720-9fcd-4bf4-97eb-bfb5a98e0239.png" + }, + { + "name": "McCheese Burger Veg Combo", + "price": "388.99", + "description": "Enjoy a deliciously filling meal of McCheese Veg Burger + Fries (M) + Beverage of your Choice in a delivery friendly, reusable bottle..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/22b25dd0-80ed-426b-a323-82c6b947612d_df827568-54f9-4329-af60-1fff38d105e9.png" + }, + { + "name": "McSpicy Premium Burger Chicken Combo", + "price": "398.99", + "description": "A deliciously filling meal of McSpicy Premium Chicken Burger + Fries (M) + Drink of your choice.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/28550356-fea3-47ce-bfa8-11222c78958a_8975c012-a121-42c3-92e7-719644761d83.png" + }, + { + "name": "McSpicy Premium Burger Veg Combo", + "price": "384.99", + "description": "A deliciously filling meal of McSpicy Premium Veg Burger + Fries (M) + Drink of your choice.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/14ce2cab-913c-4916-b98a-df2e5ea838ff_6d6208ca-4b76-4b87-a3fe-72c2db1081d8.png" + }, + { + "name": "McCheese Burger Chicken Combo", + "price": "388.99", + "description": "Enjoy a deliciously filling meal of McCheese Chicken Burger + Fries (M) + Beverage of your Choice in a delivery friendly, reusable bottle..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/b32eb9b3-7873-4f00-b601-0c92dd5a71ef_ec318a73-0e41-4277-8606-dc5c33b04533.png" + }, + { + "name": "McAloo Tikki Burger Combo", + "price": "204.99", + "description": "Enjoy a delicious combo of McAloo Tikki Burger + Fries (M) + Beverage of your choice in a new, delivery friendly, reusable bottle..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/b03a3ad7212fca0da40e90eed372ced9" + }, + { + "name": "McCheese Burger Veg Combo with Corn", + "price": "415.99", + "description": "Enjoy a combo of McCheese Burger Veg, Classic corn, McFlurry Oreo (Small) with a beverage of your choice in a delivery friendly, resuable bottle..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/9068eeb6-c774-4354-8c18-bf2ddcc94d10_631c39d1-d78c-4275-a14f-60670ce5aee4.png" + }, + { + "name": "2 Pc Chicken Nuggets Happy Meal", + "price": "211.42", + "description": "Enjoy a combo of 2 Pc Chicken Nuggets + Sweet Corn+ B Natural Mixed Fruit Beverage + Book.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/ac8a1d83-59c4-438f-bccb-edd601dacbf5_872c3881-77a1-43f7-82ca-2a929886045d.png" + }, + { + "name": "4 Pc Chicken Nuggets Happy Meal", + "price": "259.42", + "description": "Enjoy a combo of 4 Pc Chicken Nuggets + Sweet Corn+ B Natural Mixed Fruit Beverage + Book.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/fab04bb4-4f67-459a-85ef-5e9ae7861c88_f6761c27-26b3-4456-8abb-8f6c54274b34.png" + }, + { + "name": "Chicken McNuggets 6 Pcs Combo", + "price": "350.99", + "description": "Enjoy your favorite Chicken McNuggets + Fries (M) + Drink of your choice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6e7f9411ed67fe8d8873734af1e8d4e9" + }, + { + "name": "Chicken Surprise Burger + 4 Pc Chicken McNuggets + Coke", + "price": "259.99", + "description": "Enjoy the newly launched Chicken Surprise Burger with 4 Pc Chicken McNuggets and Coke.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/4/6c0288a4-943a-4bb6-a810-b14877c0ea8f_f56187da-4ae4-4a88-b1a5-e48e28d78087.png" + }, + { + "name": "Chicken Surprise Burger Combo", + "price": "238.09", + "description": "Chicken Surprise Burger + Fries (M) + Drink of your choice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/cb9833dd-5983-4445-bb1b-8d0e70b6c930_8928ae23-ddae-4d9a-abc5-e887d7d7868e.png" + }, + { + "name": "Crispy Veggie Burger Meal (M)", + "price": "326.99", + "description": "A flavorful patty with 7 premium veggies, zesty cocktail sauce, and soft buns, paired with crispy fries (M) and a refreshing Coke (M). A perfectly satisfying and full-flavored meal!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/eb4650ba-e950-4953-b1a7-c03a691ac0d2_6ca20c30-4c62-462a-b46d-9d1f21786b49.png" + }, + { + "name": "Mc Crispy Chicken Burger Meal (M)", + "price": "366.99", + "description": "A crunchy, golden chicken thigh fillet with fresh lettuce and creamy pepper mayo between soft, toasted premium buns, served with crispy fries (M) and a refreshing Coke (M). A perfectly satisfying and full-flavored meal!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/eedf0540-f558-4945-b996-994b1cd58048_d9af8cc8-384e-48c3-ab59-2facadfe574a.png" + }, + { + "name": "Choco Crunch Cookie + McAloo Tikki Burger + Lemon Ice Tea", + "price": "284.76", + "description": "Indulge in the perfect combo,crispy Choco Crunch Cookie, classic Aloo Tikki Burger, and refreshing Lemon Iced Tea. A delicious treat for your cravings, delivered fresh to your doorstep!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/b8ed02df-fb8f-4406-a443-398731aa9ef3_dafd8af6-e740-4987-8f4a-7bdc51dda4d8.png" + }, + { + "name": "Veg Pizza McPuff + Choco Crunch Cookie + Americano", + "price": "284.76", + "description": "A delightful trio, savoury Veg Pizza McPuff, crunchy Choco Crunch Cookie, and bold Americano, perfect for a satisfying snack break!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/9d81dad2-06a4-4402-9b07-2ed418963e16_f256697b-a444-4dfa-a9ff-d6448bb67b4b.png" + }, + { + "name": "Big Yummy Cheese Meal (M)", + "price": "424.76", + "description": "Double the indulgence, double the flavor: our Big Yummy Cheese Burger meal layers a spicy paneer patty and Cheese patty with crisp lettuce and smoky chipotle sauce, served with fries (M) and a beverage of your choice.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/cb892b9e-48b5-4127-b84f-b5e395961ddf_755bf2c0-e30a-4185-81e7-c35cbd07f483.png" + }, + { + "name": "Big Yummy Chicken Meal (M)", + "price": "424.76", + "description": "Indulge in double the delight: our Big Yummy Chicken Burger meal pairs the tender grilled chicken patty and Crispy chicken patty with crisp lettuce, jalapeños, and bold chipotle sauce, served with fries (M) and a beverage of your choice ..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/93313e0c-da3b-4490-839e-fd86b84c2644_b0076aa7-ef47-4f13-b72a-40f64744db95.png" + }, + { + "name": "McSpicy Chicken Double Patty Burger", + "price": "278.19", + "description": "Indulge in our signature tender double chicken patty, coated in spicy, crispy batter, topped with creamy sauce, and crispy lettuce.. Contains: Soybeans, Milk, Egg, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/314b5b5786f73746de4880602723a913" + }, + { + "name": "McChicken Double Patty Burger", + "price": "173.24", + "description": "Enjoy the classic, tender double chicken patty with creamy mayonnaise and lettuce in every bite. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/af88f46a82ef5e6a0feece86c349bb00" + }, + { + "name": "McVeggie Double Patty Burger", + "price": "186.12", + "description": "Savour your favorite spiced double veggie patty, lettuce, mayo, between toasted sesame buns in every bite. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/2d5062832f4d36c90e7dfe61ef48e85a" + }, + { + "name": "Mexican McAloo Tikki Double Patty Burger", + "price": "93.05", + "description": "A fusion of International taste combined with your favourite aloo tikki now with two patties. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/cda0e2d51420a95fad28ad728914b6de" + }, + { + "name": "McAloo Tikki Double Patty Burger", + "price": "88.11", + "description": "The World's favourite Indian burger! A crispy double Aloo patty, tomato mayo sauce & onions. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/ef569f74786e6344883a1decdd193229" + }, + { + "name": "McAloo Tikki Burger", + "price": "69.30", + "description": "The World's favourite Indian burger! A crispy Aloo patty, tomato mayo sauce & onions. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/b13811eeee71e578bc6ca89eca0ec87f" + }, + { + "name": "Big Spicy Paneer Wrap", + "price": "239.58", + "description": "Rich & filling cottage cheese patty coated in spicy crispy batter, topped with tom mayo sauce wrapped with lettuce, onions, tomatoes & cheese.. Contains: Sulphite, Soybeans, Peanut, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/22/198c3d14-3ce8-4105-8280-21577c26e944_779c4353-2f85-4515-94d2-208e90b830eb.png" + }, + { + "name": "McSpicy Chicken Burger", + "price": "226.71", + "description": "Indulge in our signature tender chicken patty, coated in spicy, crispy batter, topped with creamy sauce, and crispy lettuce.. Contains: Soybeans, Egg, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/dcdb436c-7b9f-4667-9b73-b8fa3215d7e2_9730340a-661b-49f4-a7d9-a8a89ffe988f.png" + }, + { + "name": "McSpicy Paneer Burger", + "price": "225.72", + "description": "Indulge in rich & filling spicy paneer patty served with creamy sauce, and crispy lettuce—irresistibly satisfying!. Contains: Sulphite, Soybeans, Peanut, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/fb912d21-ad9d-4332-b7cf-8f65d69e2c47_1fa5998c-b486-449a-9a88-2e61cf92ff77.png" + }, + { + "name": "Mexican McAloo Tikki Burger", + "price": "75.42", + "description": "Your favourite McAloo Tikki with a fusion spin with a Chipotle sauce & onions. Contains: Sulphite, Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/167aeccf27bab14940fa646c8328b1b4" + }, + { + "name": "McVeggie Burger", + "price": "153.44", + "description": "Savour your favorite spiced veggie patty, lettuce, mayo, between toasted sesame buns in every bite. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/2cf63c01-fef1-49b6-af70-d028bc79be7b_bfe88b73-33a9-489c-97f3-fb24631de1fc.png" + }, + { + "name": "McEgg Burger", + "price": "69.30", + "description": "A steamed egg, spicy Habanero sauce, & onions on toasted buns, a protein packed delight!. Contains: Soybeans, Milk, Egg, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/265c57f68b1a52f1cc4b63acf082d611" + }, + { + "name": "Veg Maharaja Mac Burger", + "price": "246.51", + "description": "Savor our filling 11 layer burger! Double the indulgence with 2 corn & cheese patties, along with jalapeños, onion, cheese, tomatoes, lettuce, and spicy Cocktail sauce. . Contains: Sulphite, Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/06354d09-be1b-406c-86b5-49dc9b5062d1_2f80f39e-c951-4ca6-8fca-a243a18c3448.png" + }, + { + "name": "McChicken Burger", + "price": "150.47", + "description": "Enjoy the classic, tender chicken patty with creamy mayonnaise and lettuce in every bite. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/c093ba63-c4fe-403e-811a-dc5da0fa6661_f2ad8e1e-5162-4cf8-8dab-5cc5208cdb85.png" + }, + { + "name": "Grilled Chicken & Cheese Burger", + "price": "172.25", + "description": "A grilled chicken patty, topped with sliced cheese, spicy Habanero sauce, with some heat from jalapenos & crunch from onions. Contains: Sulphite, Soybeans, Egg, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/55a77d9e-cc28-4853-89c8-1ba3861f38c4_a378aafc-62b3-4328-a255-0f35f810966e.png" + }, + { + "name": "Corn & Cheese Burger", + "price": "166.32", + "description": "A juicy corn and cheese patty, topped with extra cheese, Cocktail sauce, with some heat from jalapenos & crunch from onions. Contains: Sulphite, Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/cb4d60c0-72c0-4694-8d41-3c745e253ea6_8262ec8c-e2ac-4c4f-8e52-36144a372851.png" + }, + { + "name": "Big Spicy Chicken Wrap", + "price": "240.57", + "description": "Tender and juicy chicken patty coated in spicy, crispy batter, topped with a creamy sauce, wrapped with lettuce, onions, tomatoes & cheese. A BIG indulgence.. Contains: Soybeans, Egg, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/22/e1fa4587-23ac-4613-af61-de659b066d19_14205d12-ea39-4894-aa85-59035020cecd.png" + }, + { + "name": "McCheese Burger Chicken", + "price": "282.15", + "description": "Double the indulgence with a sinfully oozing cheesy patty & flame-grilled chicken patty, along with chipotle sauce, shredded onion, jalapenos & lettuce.. Contains: Sulphite, Soybeans, Egg, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/b1184d5f-0785-4393-98a8-a712d280a045_027d0e63-e5d9-43f3-9c9c-0dc68dd1ece1.png" + }, + { + "name": "McCheese Burger Veg", + "price": "262.35", + "description": "Find pure indulgence in our Veg McCheese Burger, featuring a sinfully oozing cheesy veg patty, roasted chipotle sauce, jalapenos & lettuce.. Contains: Sulphite, Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/de84d4cc-6169-4235-942e-e4883a81c2e0_d90762c6-283f-46e5-a6ed-14ee3262bae0.png" + }, + { + "name": "McSpicy Premium Chicken Burger", + "price": "259.38", + "description": "A wholesome Spicy Chicken patty, Lettuce topped with Jalapenos and Cheese slice, Spicy Cocktail sauce & Cheese sauce. Contains: Sulphite, Soybeans, Egg, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/e1c10ab2-671b-4eac-aee8-88d9f96e005b_40169ccb-b849-4e0d-a54a-8938dd41ea34.png" + }, + { + "name": "McSpicy Premium Veg Burger", + "price": "249.47", + "description": "A wholesome Spicy Paneer patty, Lettuce topped with Jalapenos and Cheese slice, Spicy Cocktail sauce & Cheese sauce. Contains: Sulphite, Soybeans, Peanut, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/1a9faafd-b523-40a2-bdc6-f35191cfcf4a_0c462cb4-3843-4997-b813-dc34249b7c91.png" + }, + { + "name": "Chicken Maharaja Mac Burger", + "price": "268.28", + "description": "Savor our filling 11 layer burger! Double the indulgence with 2 juicy grilled chicken patties, along with jalapeños, onion, cheese, tomatoes, lettuce, and zesty Habanero sauce. . Contains: Sulphite, Soybeans, Egg, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/65a88cf4-7bcd-40f6-a09d-ec38c307c5d9_8993ea5b-f8e0-4d6c-9691-7f85adee2000.png" + }, + { + "name": "Chicken Surprise Burger", + "price": "75.23", + "description": "Introducing the new Chicken Surprise Burger which has the perfect balance of a crispy fried chicken patty, the crunch of onions and the richness of creamy sauce.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/0fbf18a1-5191-4cda-a09d-521a24c8c6ca_25cf57c6-48cc-47bd-b422-17e86b816422.png" + }, + { + "name": "McAloo Tikki Burger NONG", + "price": "69.30", + "description": "The World's favourite Indian burger with No Onion & No Garlic! Crispy aloo patty with delicious Tomato Mayo sauce!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/60311aec-07af-483d-b66a-ecae76edbd75_1407e287-7f07-4717-a6b1-14bff1a34961.png" + }, + { + "name": "Mexican McAloo Tikki Burger NONG", + "price": "74.24", + "description": "Your favourite McAloo Tikki with a fusion spin of Chipotle sauce. No Onion and No Garlic.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/c7d3076a-dfa7-4725-89cd-53754cecebee_1c11b7da-e00d-4e6b-bbe8-8ee847ee88a1.png" + }, + { + "name": "Crispy Veggie Burger", + "price": "198", + "description": "A flavorful patty made with a blend of 7 premium veggies, topped with zesty cocktail sauce, all served between soft, premium buns. Perfectly satisfying and full of flavor.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/7a3244bf-3091-4ae6-92e3-13be841a753e_b21f7a05-24b3-43f8-a592-beb23e6b69fa.png" + }, + { + "name": "Mc Crispy Chicken Burger", + "price": "221.76", + "description": "A crunchy, golden chicken thigh fillet, topped with fresh lettuce and creamy pepper mayo, all nestled between soft, toasted premium buns. Perfectly satisfying and full of flavor.. Contains: Gluten, Milk, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/df040551-263c-4074-86ee-68cb8cd393ba_3a5e53c6-601e-44a8-bf2e-590bffd7ee5e.png" + }, + { + "name": "McAloo Tikki Burger with Cheese", + "price": "98.05", + "description": "Savor the classic McAloo Tikki Burger, with an add-on cheese slice for a cheesy indulgence.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/10/30/0f0cbfd7-ee83-4e7c-bb6d-baab81e51646_4746c010-f82c-4e23-a43f-e36fe50b883b.png" + }, + { + "name": "Mexican McAloo Tikki with Cheese", + "price": "98.05", + "description": "Savor your favourite Mexican McAloo Tikki Burger, with an add-on cheese slice for a cheesy indulgence.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/10/30/d3cfbc30-4a7c-40a8-88a9-f714f3475523_14e7323b-24fe-4fbf-a517-68de64489103.png" + }, + { + "name": "Big Yummy Cheese Burger.", + "price": "349", + "description": "A spicy, cheesy indulgence, the Big Yummy Cheese Burger stacks a fiery paneer patty and a rich McCheese patty with crisp lettuce and smoky chipotle sauce on a Quarter Pound bun.. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/5c65bd4d-86ff-424c-800c-7d8b04aac86d_99e0d34c-cbba-41d9-bc31-a093237ba2af.png" + }, + { + "name": "Big Yummy Chicken Burger.", + "price": "349", + "description": "Crafted for true indulgence, tender grilled chicken patty meets the McCrispy chicken patty, elevated with crisp lettuce, jalapenos, and bold chipotle sauce.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/33234ab9-f10d-4605-89fc-349e0c7058bf_edba8f21-7c28-4fb6-88a2-dd3a86966175.png" + }, + { + "name": "Fries (Regular)", + "price": "88.11", + "description": "World Famous Fries, crispy, golden, lightly salted and fried to perfection! Also known as happiness.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/5a18fbbff67076c9a4457a6b220a55d9" + }, + { + "name": "Fries (Large)", + "price": "140.58", + "description": "World Famous Fries, crispy, golden, lightly salted and fried to perfection! Also known as happiness.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/a4b3002d0ea35bde5e5983f40e4ebfb4" + }, + { + "name": "Tomato Ketchup Sachet", + "price": "1", + "description": "Looking for a sauce to complement your meal? Look no further.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7db5533db29a4e9d2cc033f35c5572bc" + }, + { + "name": "9 Pc Chicken Nuggets", + "price": "221.74", + "description": "9 pieces of our iconic crispy, golden fried Chicken McNuggets!. Contains: Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/1ca7abb262e8880f5cb545d0d2f9bb9b" + }, + { + "name": "6 Pc Chicken Nuggets", + "price": "183.14", + "description": "6 pieces of our iconic crispy, golden fried Chicken McNuggets!. Contains: Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/44dc10c1099d7c366db9f5ce776878bd" + }, + { + "name": "Piri Piri Spice Mix", + "price": "23.80", + "description": "The perfect, taste bud tingling partner for our World Famous Fries. Shake Shake, and dive in!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/df3edfc74f610edff535324cc53a362a" + }, + { + "name": "Fries (Medium)", + "price": "120.78", + "description": "World Famous Fries, crispy, golden, lightly salted and fried to perfection! Also known as happiness.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/8a61e7fd97c454ea14d0750859fcebb8" + }, + { + "name": "Chilli Sauce Sachet", + "price": "2", + "description": "Looking for a sauce to complement your meal? Look no further.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f708dfc29c9624d8aef6e6ec30bde1c9" + }, + { + "name": "Veg Pizza McPuff", + "price": "64.35", + "description": "Crispy brown crust with a generous filling of rich tomato sauce, mixed with carrots, bell peppers, beans, onions and mozzarella. Served HOT.. Contains: Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/abe4b8cdf0f1bbfd1b9a7a05be3413e8" + }, + { + "name": "Classic Corn Cup", + "price": "90.08", + "description": "A delicious side of golden sweet kernels of corn in a cup.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9d67eae020425c4413acaf5af2a29dce" + }, + { + "name": "Fries (M) + Piri Piri Mix", + "price": "125", + "description": "Flat 15% Off on Fries (M) + Piri Piri Mix.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/15/b02b9b46-0d2b-46a6-aaa3-171c35101e11_24766c28-4fe9-4562-92d3-85c39d29c132.png" + }, + { + "name": "Cheesy Fries", + "price": "157.40", + "description": "The world famous, crispy golden Fries, served with delicious cheese sauce with a hint of spice. Contains cheese & mayonnaise. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/086cb28d-501d-42e5-a603-33e2d4493588_11348186-570f-44b8-b24e-88855455ba25.png" + }, + { + "name": "20 Pc Chicken Nuggets", + "price": "445.97", + "description": "20 pieces of our iconic crispy, golden fried Chicken McNuggets!.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/1ca7abb262e8880f5cb545d0d2f9bb9b" + }, + { + "name": "Barbeque Sauce", + "price": "19.04", + "description": "Looking for a sauce to complement your meal? Look no further.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/ba0a188d45aecc3d4d187f340ea9df54" + }, + { + "name": "4 Pcs Chicken Nuggets", + "price": "109.88", + "description": "4 pieces of our iconic crispy, golden fried Chicken McNuggets!. Contains: Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/44dc10c1099d7c366db9f5ce776878bd" + }, + { + "name": "Mustard Sauce", + "price": "19.04", + "description": "Looking for a sauce to complement your meal? Look no further.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6c3aeffdbd544ea3ceae1e4b8ce3fc43" + }, + { + "name": "Spicy Sauce", + "price": "33.33", + "description": "Enjoy this spicy sauce that will add an extra kick to all your favourite items.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/dcc9ef6b-ceda-4b15-87af-ba2ad6c7de28_cfa5487a-811c-4c41-a770-af4ba6c5ebc8.png" + }, + { + "name": "Mango Smoothie", + "price": "210.86", + "description": "A delicious mix of mangoes, soft serve mix and blended ice. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/bb8470e1-5862-4844-bb47-ec0b2abdd752_845f1527-6bb0-46be-a1be-2fcc7532e813.png" + }, + { + "name": "Strawberry Green Tea (S)", + "price": "153.44", + "description": "Freshly-brewed refreshing tea with fruity Strawbery flavour.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/167dc0134bc1f4e8d7cb8e5c2a9dde5d" + }, + { + "name": "Moroccan Mint Green Tea (R )", + "price": "203.94", + "description": "Freshly-brewed refreshing tea with hint of Moroccon Mint flavour.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/2699c7c2130b4f50a09af3e294966b2e" + }, + { + "name": "Americano Coffee (R)", + "price": "190.07", + "description": "Refreshing cup of bold and robust espresso made with our signature 100% Arabica beans, combined with hot water.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/6a300499-efad-4a89-8fe4-ef6f6d3c1e2d_db248587-e3ea-4034-bbd7-33f9e483d6cb.png" + }, + { + "name": "Mixed Berry Smoothie", + "price": "210.86", + "description": "A mix of mixed berries, blended together with our creamy soft serve. Contains: Sulphite, Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/7a70ab81-ef1e-4e13-9c78-43ed2c62eaeb_3cd64c86-3a7b-41a6-9528-5fb5e2bbadc7.png" + }, + { + "name": "McCafe-Ice Coffee", + "price": "202.95", + "description": "Classic coffee poured over ice with soft servefor a refreshing pick-me-up. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/14/fd328038-532b-4df7-b6e9-b21bd6e8c70f_7e663943-ab8a-4f24-8dc0-d727dd503cd3.png" + }, + { + "name": "American Mud Pie Shake", + "price": "202.95", + "description": "Creamy and rich with chocolate and blended with nutty brownie bits for that extra thick goodness. Contains: Soybeans, Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/5ad3343d-a520-498d-a933-52104c624304_9f6cb499-0229-40dc-8b17-4c386f0cc287.png" + }, + { + "name": "Ice Americano Coffee", + "price": "180.18", + "description": "Signature Arabica espresso shot mixed with ice for an energizing experience.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7d89db9d67c537d666d838ddc1e0c44f" + }, + { + "name": "Americano Coffee (S)", + "price": "170.27", + "description": "Refreshing cup of bold and robust espresso made with our signature 100% Arabica beans, combined with hot water.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/6a300499-efad-4a89-8fe4-ef6f6d3c1e2d_db248587-e3ea-4034-bbd7-33f9e483d6cb.png" + }, + { + "name": "Coke", + "price": "101.97", + "description": "The perfect companion to your burger, fries and everything nice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/a1afed29afd8a2433b25cc47b83d01da" + }, + { + "name": "Mocha Coffee (R)", + "price": "236.60", + "description": "A delight of ground Arabica espresso, chocolate syrup and steamed milk. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/04a137420bf3febf861c4beed86d5702" + }, + { + "name": "Mocha Coffee (S)", + "price": "210.86", + "description": "A delight of ground Arabica espresso, chocolate syrup and steamed milk. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/71df07eb87d96824e2122f3412c8f743" + }, + { + "name": "Hot Chocolate (R)", + "price": "224.73", + "description": "Sinfully creamy chocolate whisked with silky streamed milk. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/1948679f-65df-4f93-a9cc-2ec796ca0818_6ad8b55d-e3cc-48ed-867d-511100b5735d.png" + }, + { + "name": "Latte Coffee (R)", + "price": "202.95", + "description": "A classic combination of the signature McCafe espresso, smooth milk, steamed and frothed. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/cee1ec0e10e25018572adcaf3a3c9e8c" + }, + { + "name": "Coke zero can", + "price": "66.66", + "description": "The perfect diet companion to your burger, fries and everything nice. Regular serving size, 300 Ml..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/8d6a37c4fc69bceb66b6a66690097190" + }, + { + "name": "Schweppes Water bottle", + "price": "66.66", + "description": "Quench your thirst with the Schweppes Water bottle.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/6/df44cf7c-aa13-46e2-9b38-e95a2f9faed4_d6c65089-cd93-473b-b711-aeac7fcf58b0.png" + }, + { + "name": "Fanta", + "price": "101.97", + "description": "Add a zest of refreshing orange to your meal..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7d662c96bc13c4ac33cea70c691f7f28" + }, + { + "name": "Mixed Fruit Beverage", + "price": "76.19", + "description": "Made with puree, pulp & juice from 6 delicious fruits. Contains: Soybeans, Peanut, Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/8e455d39bbbd8e4107b2099da51f3933" + }, + { + "name": "Sprite", + "price": "101.97", + "description": "The perfect companion to your burger, fries and everything nice..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/46e03daf797857bfbce9f9fbb539a6aa" + }, + { + "name": "Cappuccino Coffee (R)", + "price": "199.98", + "description": "A refreshing espresso shot of 100% Arabica beans, topped with steamed milk froth. 473ml. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/4f45f0d8-111a-4d9c-a993-06d6858fdb06_cb5d79e9-c112-45e5-9e6a-e17d75acd8ac.png" + }, + { + "name": "Cappuccino Coffee (S)", + "price": "170.27", + "description": "A refreshing espresso shot of 100% Arabica beans, topped with steamed milk froth. 236ml. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/4f45f0d8-111a-4d9c-a993-06d6858fdb06_cb5d79e9-c112-45e5-9e6a-e17d75acd8ac.png" + }, + { + "name": "Berry Lemonade Regular", + "price": "141.57", + "description": "A refreshing drink, made with the delicious flavors of berries. 354 ml..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/14/4f0f852a-1e4a-4bd9-b0ec-69ccef3c6a34_0f4f0b64-1b1b-4a7d-98f9-e074fc96d1bd.png" + }, + { + "name": "Chocolate Flavoured Shake", + "price": "183.15", + "description": "The classic sinful Chocolate Flavoured Shake, a treat for anytime you need one. Now in new, convenient and delivery friendly packaging. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/463331b6-aa39-4d37-a8b2-c175aa20f723_14e888b6-2ebf-4d84-93a7-7b0576147bda.png" + }, + { + "name": "McCafe-Classic Coffee", + "price": "214.82", + "description": "An irrestible blend of our signature espresso and soft serve with whipped cream on top, a timeless combination! Now in a new, convenient and delivery friendly packaging.. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/5f9bdb36689a11cadb601a27b6fdef2d" + }, + { + "name": "Mocha Frappe", + "price": "278.19", + "description": "The perfect mix of indulgence and bold flavours. Enjoy a delicious blend of coffee, chocolate sauce, and soft serve. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/063e9cd747c621978ab4fddbb6d0a5ee" + }, + { + "name": "Cappuccino Small with Hazelnut", + "price": "183.15", + "description": "A delightful and aromatic coffee beverage that combines the robust flavor of espresso with the rich, nutty essence of hazelnut.. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/1793c61c-48f4-4785-b8a6-bc50eadb3b88_36ab0b13-0b17-459c-97a6-50dea3027b80.png" + }, + { + "name": "Ice Tea - Green Apple flavour", + "price": "182.16", + "description": "A perfect blend of aromatic teas, infused with green apple flavour .Now in a new, convenient and delivery friendly packaging.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/54243043b1b31fa715f38e6998a63e93" + }, + { + "name": "Strawberry Shake", + "price": "183.15", + "description": "An all time favourite treat bringing together the perfect blend of creamy vanilla soft serve and strawberry flavor.Now in new, convenient and delivery friendly packaging. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/b7d85483-4328-40a7-8cba-c408771d2482_99cab512-28fc-4b2c-b07f-87850002e79c.png" + }, + { + "name": "Cappuccino Small with French Vanilla", + "price": "182.16", + "description": "A popular coffee beverage that combines the smooth, creamy flavor of vanilla with the robust taste of espresso. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/e0a5c619-2df5-431f-9e21-670a358e8dbb_c2a2324f-ff95-44b3-adf9-dd6aee9696aa.png" + }, + { + "name": "Classic Coffee Regular with French Vanilla", + "price": "236", + "description": "a delightful and refreshing beverage that blends into the smooth, creamy essence of vanilla with the invigorating taste of chilled coffee.. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/5926eaba-bd6a-4af1-ba57-89bdb2fdfec6_c2507538-7597-4fc0-92e3-bdf57c81bda5.png" + }, + { + "name": "Classic Coffee Regular with Hazelnut", + "price": "233.63", + "description": "refreshing and delicious beverage that combines the rich, nutty taste of hazelnut with the cool, invigorating essence of cold coffee. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/b65446d8-bee6-484c-abdf-0326315c7e40_4c694dbc-e5e9-4998-b6ac-bd280fc6b5dc.png" + }, + { + "name": "Iced Coffee with French Vanilla", + "price": "223.74", + "description": "An ideal choice for those who enjoy a smooth, creamy vanilla twist to their iced coffee, providing a satisfying and refreshing pick-me-up. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/35867352-7802-4082-8b25-955878a6daa2_1d995db8-655f-45d3-9ad2-4469fe1301ae.png" + }, + { + "name": "Iced Coffee with Hazelnut", + "price": "223.74", + "description": "An ideal choice for those who enjoy a flavorful, nutty twist to their iced coffee, providing a satisfying and refreshing pick-me-up.. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/0619409e-e7cc-4b7c-a002-c05a2dff2ed8_f9bf368f-3dae-4af5-813d-4645384d24f7.png" + }, + { + "name": "Mint Lime Cooler", + "price": "101.97", + "description": "Refresh your senses with our invigorating Mint Lime Cooler. This revitalizing drink combines the sweetness of fresh lime juice and the subtle tang, perfectly balanced to quench your thirst and leave you feeling revitalized. Contains: Gluten, Milk, Peanut, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/a6b79a50-55c4-4b83-8b3c-d59c86ee1337_a3063ad4-fee2-4926-bf6d-ec98051b2249.png" + }, + { + "name": "Choco Crunch Cookie", + "price": "95", + "description": "Grab the choco crunch cookies packed with chocolate chips for the perfect crunch. Contains: Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/32ff88484a8607b6d740c1635b9ce09f" + }, + { + "name": "Hot Coffee Combo", + "price": "181", + "description": "In this combo choose any 1 hot coffee among- Cappuccino(s)/Latte(s)/Mocha(s)/Americano(s) and any 1 snacking item among- Choco crunch cookies/Oats Cookies/Choco Brownie/Blueberry muffin/ Signature croissant.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9aba6528e9c4f780dda18b5068349020" + }, + { + "name": "Indulge Choco Jar Dessert", + "price": "76", + "description": "Rich chocolate for pure indulgence to satify your sweet tooth..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/c7165efd872543e1648c21c930dafe5f" + }, + { + "name": "Cinnamon Raisin Cookie", + "price": "95", + "description": "Enjoy the wholesome flavours of this chewy and satisfying cookie. Contains: Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/fa5526832dfc6e236e6de7c322beae94" + }, + { + "name": "Cold Coffee Combo", + "price": "185", + "description": "In this combo choose any 1 coffee among- Cold Coffee ( R )/Iced Coffee ( R )/Iced Americano ( R ) and any 1 snacking item among- Choco crunch cookies/Oats Cookies/Choco Brownie/Blueberry muffin.", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/cb24719daf96b58dd1be3bbf7b9fe372" + }, + { + "name": "Chocochip Muffin", + "price": "142", + "description": "Enjoy a dense chocochip muffin, with melty chocolate chips for a choco-lover's delight. Contains: Milk", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6c748c0b21f4a99593d1040021500430" + }, + { + "name": "Indulge Combo", + "price": "171", + "description": "Indulge in the perfect pairing of a classic cold coffee and a chocolate chip muffin, that balances the refreshing taste of chilled coffee with the sweet, comforting flavors of a freshly baked treat..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/ee2c7b9f-4426-4e70-9e98-bf8e8cc754ad_93eaaaa3-e498-4ca1-a77d-545a8006c8e2.png" + }, + { + "name": "Take a break Combo", + "price": "171", + "description": "Savor the harmonious pairing of a classic cappuccino with a cinnamon cookie, combining the bold, creamy flavors of coffee with the warm, spiced sweetness of a baked treat .", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/87a8807b-77e4-4e0d-a47e-af1cf2d2b96d_3230e601-af8f-4436-85cd-14c4e855240d.png" + }, + { + "name": "Treat Combo", + "price": "171", + "description": "Delight in the luxurious pairing of a chocolate jar dessert with a classic cappuccino, combining rich, creamy indulgence with the bold, aromatic flavors of espresso..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/9462ac0c-840e-44d1-a85d-bbea6f8268f1_a93dc44b-bf30-47b9-8cee-3a5589d2002e.png" + }, + { + "name": "Butter Croissant", + "price": "139", + "description": "Buttery, flaky croissant baked to golden perfection.Light, airy layers with a crisp outer shell.A classic French treat that melts in your mouth..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/da6cbf69-e422-4d02-9cc9-cb0452d04874_2153fe0c-389e-4842-b4c9-e0b78c450625.png" + }, + { + "name": "Butter Croissant + Cappuccino", + "price": "209", + "description": "Buttery croissant paired with a rich, frothy cappuccino.Warm, comforting, and perfectly balanced.A timeless duo for your anytime cravings..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/227a6aa3-f18c-4685-bb09-a76a01ec39a1_f347d737-b634-43d7-9b73-dc99bccde65f.png" + }, + { + "name": "Butter Croissant + Iced Coffee", + "price": "209", + "description": "Buttery, flaky croissant served with smooth, refreshing iced coffee. A classic combo that's light, crisp, and energizing. Perfect for a quick, satisfying bite..", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/be3c8688-f975-4d1e-aad7-9229232bcc69_65c9a679-687d-4e66-9847-eb4d5c0f9825.png" + }, + { + "name": "Mcflurry Oreo ( S )", + "price": "104", + "description": "Delicious soft serve meets crumbled oreo cookies, a match made in dessert heaven. Perfect for one.. Contains: Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/a28369e386195be4071d9cf5078a438d" + }, + { + "name": "McFlurry Oreo ( M )", + "price": "129", + "description": "Delicious soft serve meets crumbled oreo cookies, a match made in dessert heaven. Share it, if you can.. Contains: Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f966500ed8b913a16cfdb25aab9244e4" + }, + { + "name": "Hot Fudge Sundae", + "price": "66", + "description": "A sinful delight, soft serve topped with delicious, gooey hot chocolate fudge. Always grab an extra spoon.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9c8958145495e8f2cf70470195f7834a" + }, + { + "name": "Strawberry Sundae", + "price": "66", + "description": "The cool vanilla soft serve ice cream with twirls of strawberry syrup.. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/d7bd22aa47cffdcdde2d5b6223fde06e" + }, + { + "name": "Oreo Sundae ( M )", + "price": "72", + "description": "Enjoy the classic McFlurry Oreo goodness with a drizzle of hot fudge sauce with the Oreo Sundae!. Contains: Soybeans, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/3696da86802f534ba9ca68bd8be717ab" + }, + { + "name": "Black Forest McFlurry Medium", + "price": "139", + "description": "A sweet treat to suit your every mood. Contains: Soybeans, Peanut, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f513cc8c35cadd098835fb5b23c03561" + }, + { + "name": "Hot Fudge Brownie Sundae", + "price": "139", + "description": "Luscious chocolate brownie and hot-chocolate fudge to sweeten your day. Contains: Soybeans, Peanut, Milk, Gluten", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6e00a57c6d8ceff6812a765c80e9ce74" + }, + { + "name": "Chocolate Overload McFlurry with Oreo Medium", + "price": "164.76", + "description": "Indulge in your chocolatey dreams with creamy soft serve, Oreo crumbs, a rich Hazelnut brownie, and two decadent chocolate sauces. Tempting, irresistible, and unforgettable. Contains: Gluten, Milk, Peanut, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/eb789769-9e60-4a84-9c64-90887ca79d7c_86154b6a-146c-47e9-9bbe-e685e4928e2e.png" + }, + { + "name": "Chocolate Overload McFlurry with Oreo Small", + "price": "134.28", + "description": "Indulge in your chocolatey dreams with creamy soft serve, Oreo crumbs, a rich Hazelnut brownie, and two decadent chocolate sauces. Tempting, irresistible, and unforgettable. Contains: Gluten, Milk, Peanut, Soybeans", + "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/431024a3-945d-4e6b-aafe-d35c410ac513_1b01e6dc-90cc-411d-a75e-9b98b42e178d.png" + } +] \ No newline at end of file From 564d437d97492057143716d1f3787eb1f4e5b81f Mon Sep 17 00:00:00 2001 From: Aravind Karnam Date: Fri, 17 Oct 2025 15:31:29 +0530 Subject: [PATCH 068/119] docs: fix order of star history and Current sponsors --- c4ai_menu.json | 1688 ------------------------------------------- firecrawl_menu.json | 28 - menu.json | 1688 ------------------------------------------- 3 files changed, 3404 deletions(-) delete mode 100644 c4ai_menu.json delete mode 100644 firecrawl_menu.json delete mode 100644 menu.json diff --git a/c4ai_menu.json b/c4ai_menu.json deleted file mode 100644 index 810be887..00000000 --- a/c4ai_menu.json +++ /dev/null @@ -1,1688 +0,0 @@ -[ - { - "name": "Big Yummy Cheese Burger", - "price": "349", - "description": "A spicy, cheesy indulgence, the Big Yummy Cheese Burger stacks a fiery paneer patty and a rich McCheese patty with crisp lettuce and smoky chipotle sauce on a Quarter Pound bun.. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/916ad567-194a-449a-a9f3-c53acc0fa52e_4dfe7bfa-a200-4eab-9ffe-532236399652.png" - }, - { - "name": "Big Yummy Cheese Meal (M).", - "price": "424.76", - "description": "Double the indulgence, double the flavor: our Big Yummy Cheese Burger meal layers a spicy paneer patty and Cheese patty with crisp lettuce and smoky chipotle sauce, served with fries (M) and a beverage of your choice.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/7c1c4952-781a-4fe7-ad31-d6650fccbbdd_19f5f0fa-2cb3-405c-818a-457c84ba5a01.png" - }, - { - "name": "Big Yummy Chicken Burger", - "price": "349", - "description": "Crafted for true indulgence, tender grilled chicken patty meets the McCrispy chicken patty, elevated with crisp lettuce, jalapenos, and bold chipotle sauce.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/8bdf918d-2571-4f8c-a8f6-bcae8a257144_dbbba789-1e27-4bd0-a356-d7f5ed17c103.png" - }, - { - "name": "Big Yummy Chicken Meal (M).", - "price": "424.76", - "description": "Indulge in double the delight: our Big Yummy Chicken Burger meal pairs the tender grilled chicken patty and Crispy chicken patty with crisp lettuce, jalapeños, and bold chipotle sauce, served with fries (M) and a beverage of your choice ..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/be8e14ab-7861-4cf7-9550-d4a58f09d28c_958f0a00-5a1d-4041-825c-e196ae06d524.png" - }, - { - "name": "Cappuccino (S) + Iced Coffee (S)", - "price": "199.04", - "description": "Get the best coffee combo curated just for you!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/19/5aeea709-728c-43a2-ab00-4e8c754cec74_494372be-b929-496e-8db8-34e128746eb9.png" - }, - { - "name": "Veg Pizza McPuff + McSpicy Chicken Burger", - "price": "260", - "description": "Tender and juicy chicken patty coated in spicy, crispy batter topped with a creamy sauce and crispy shredded lettuce will have you craving for more. Served with Veg Pizza McPuff.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/fb4b9d4775505e82d05d6734ef3e2491" - }, - { - "name": "2 Cappuccino", - "price": "233.33", - "description": "2 Cappuccino (S).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/34aaf6ee-06e2-4c60-9950-8ad569bc5898_2639ffd2-47dd-447b-bbbe-eb391315666f.png" - }, - { - "name": "McChicken Burger + McSpicy Chicken Burger", - "price": "315.23", - "description": "The ultimate chicken combo made just for you. Get the top selling McChicken with the McSpicy Chicken Burger..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/006d51070b0ab9c839a293b87412541c" - }, - { - "name": "2 Iced Coffee", - "price": "233.33", - "description": "Enjoy 2 Iced Coffee.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/e36a9ed2-bfc0-4d36-bfef-297e2c74f991_90e415d8-c4f0-45c1-ad81-db8ef52a5f96.png" - }, - { - "name": "McVeggie Burger + McAloo Tikki Burger", - "price": "210.47", - "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, makes our iconic McVeggie and combo with our top selling McAloo Tikki Burger..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/1f4d583548597d41086df0c723560da7" - }, - { - "name": "Strawberry Shake + Fries (M)", - "price": "196", - "description": "Can't decide what to eat? We've got you covered. Get this snacking combo with Medium Fries and Strawberry Shake..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/74603316fc90ea3cd2b193ab491fbf53" - }, - { - "name": "McChicken Burger + Fries (M)", - "price": "244.76", - "description": "Tender and juicy chicken patty cooked to perfection, with creamy mayonnaise and crunchy lettuce adding flavour to each bite. Served with Medium Fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/a321db80-a223-4a90-9087-054154d27189_9168f1ee-991b-4d8c-8e82-64b2ea249943.png" - }, - { - "name": "McVeggie Burger + Fries (M)", - "price": "215.23", - "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns. Served with Medium Fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/d14cc495747a172686ebe43e675bc941" - }, - { - "name": "McAloo Tikki Burger + Veg Pizza McPuff + Coke", - "price": "190.47", - "description": "The ultimate veg combo made just for you. Get the top selling McAloo Tikki served with Veg Pizza McPuff and Coke..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9f0269a2d28f4918a3b07f63a487f26d" - }, - { - "name": "McAloo Tikki + Fries (R)", - "price": "115.23", - "description": "Aloo Tikki+ Fries (R).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/1ffa9f16-d7ce-48d8-946a-aaac56548c88_10b13615-190f-4f06-91cb-ac7499046fb8.png" - }, - { - "name": "Mexican McAloo Tikki Burger + Fries (R)", - "price": "120", - "description": "A fusion of international taste combined with your favourite aloo tikki patty, layered with shredded onion, and delicious Chipotle sauce. Served with Regular Fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7274d82212a758e597550e8c246fb2f7" - }, - { - "name": "McChicken Burger + Veg Pizza McPuff", - "price": "184.76", - "description": "Tender and juicy chicken patty cooked to perfection, with creamy mayonnaise and crunchy lettuce adding flavour to each bite. Served with Veg Pizza McPuff..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9a66b8ef66d780b9f83a0fc7cd434ded" - }, - { - "name": "McVeggie Burger + Veg Pizza McPuff", - "price": "195.23", - "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns. Served with Veg Pizza McPuff..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/a34eb684-5878-4578-8617-b06e24e46fba_36b736e7-3a91-4618-bf0f-ce60d45e55d2.png" - }, - { - "name": "McVeggie Burger + Fries (R)", - "price": "184.76", - "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns. Served with Regular fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/5e29050f-38f9-4d42-9388-be895f2ba84b_591326c1-a753-44b9-b80c-462196c67fd0.png" - }, - { - "name": "Mexican McAloo Tikki Burger + Fries (L)", - "price": "180", - "description": "A fusion of international taste combined with your favourite aloo tikki patty, layered with shredded onion, and delicious Chipotle sauce. Served with Large Fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f4be6e877d2567a0b585d4b16e53871e" - }, - { - "name": "McChicken Burger + Fries (L)", - "price": "250.47", - "description": "Tender and juicy chicken patty cooked to perfection, with creamy mayonnaise and crunchy lettuce adding flavour to each bite. Served with Large Fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/08e794cb-6520-4907-890c-27449181c9fb_e30fa88b-c5aa-4b71-8c84-135dc433c954.png" - }, - { - "name": "McVeggie Burger + Fries (L)", - "price": "250.47", - "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns. Served with Large fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/b02d421c-fae4-4408-870d-3b51c4e4a94d_0a64d6d8-9eea-452f-917e-988d7db846e2.png" - }, - { - "name": "2 Fries (R)", - "price": "120", - "description": "World Famous Fries, crispy, golden, lightly salted and fried to perfection! Double your happiness with this fries combo.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/4a170da5ecae92e11410a8fbb44c8476" - }, - { - "name": "2 McVeggie Burger", - "price": "270.47", - "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns makes our iconic McVeggie..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/f1edf611-08aa-4b91-b99c-f135eb70df66_ce7bec39-1633-4222-89a2-40013b5d9281.png" - }, - { - "name": "Grilled Chicken & Cheese Burger + Coke", - "price": "239.99", - "description": "Flat 15% Off on Grilled Chicken & Cheese Burger + Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/5/29/18f3ce07-00b5-487b-8608-56440efff007_2b241e0a-fad5-4be4-a848-81c124c95b8b.png" - }, - { - "name": "McAloo Tikki Burger + Veg Pizza McPuff + Fries (R)", - "price": "208.57", - "description": "Flat 15% Off on McAloo Tikki + Veg Pizza McPuff + Fries (R).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/6d7aef29-bae6-403b-a245-38c3371a5363_5dc1ef9c-1f17-4409-946b-b2d78b8710d4.png" - }, - { - "name": "McVeggie Burger + Fries (M) + Piri Piri Mix", - "price": "240", - "description": "Flat 15% Off on McVeggie Burger + Fries (M) + Piri Piri Mix.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/3011bf9d-01fb-4ff3-a2ca-832844aa0dd0_8c231aef-ff56-4dd2-82e5-743a6240c37b.png" - }, - { - "name": "McVeggie Burger + Veg Pizza McPuff + Fries (L)", - "price": "290.47", - "description": "Flat 15% Off on McVeggie Burger + Veg Pizza McPuff + Fries (L).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/5/29/0e4d8ee8-ee6b-4e12-9386-658dbcdc6be4_6810365e-3735-4b21-860b-923a416403be.png" - }, - { - "name": "6 Pc Chicken Nuggets + Fries (M) + Piri Piri Spice Mix", - "price": "247.99", - "description": "The best Non veg sides combo curated for you! Get 6 pc Chicken McNuggets + Fries M. Top it up with Piri Piri mix..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/8cba0938b4401e5c8cb2ccf3741b93c4" - }, - { - "name": "McAloo Tikki Burger + Veg Pizza McPuff + Piri Piri Spice Mix", - "price": "128", - "description": "Get India's favourite burger - McAloo Tikki along with Veg Pizza McPuff and spice it up with a Piri Piri Mix.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/46bcb1e486cbe6dcdb0487b063af58a6" - }, - { - "name": "Grilled Chicken & Cheese Burger + Veg Pizza McPuff", - "price": "196", - "description": "A delicious Grilled Chicken & Cheese Burger + a crispy brown, delicious Pizza McPuff.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/80983b8d-5d94-45f7-84a8-57f2477933de_83c6e531-c828-4ecd-a68a-ed518677fb66.png" - }, - { - "name": "Corn & Cheese Burger + Veg Pizza McPuff", - "price": "184.76", - "description": "A delicious Corn & Cheese Burger + a crispy brown, delicious Pizza McPuff.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/6d92a078-e41e-4126-b70c-b57538675892_c28b61a2-2733-4b3c-a6a2-7943fa109e24.png" - }, - { - "name": "Corn & Cheese Burger + Fries (R)", - "price": "210.47", - "description": "A delicious Corn & Cheese Burger + a side of crispy, golden, world famous fries ??.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/c66d0139-6560-4be9-9239-762f9e80a31a_b4576f89-8186-461d-ba99-28a31a0507f9.png" - }, - { - "name": "Corn & Cheese Burger + Coke", - "price": "239.99", - "description": "Flat 15% Off on Corn & Cheese Burger + Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/5/31/043061fe-2b66-49a4-bc65-b26232584003_ec753203-f35a-487f-bc87-8dcccd31ea3f.png" - }, - { - "name": "Chocolate Flavoured Shake+ Fries (M)", - "price": "196", - "description": "Can't decide what to eat? We've got you covered. Get this snacking combo with Medium Fries and Chocolate Flavoured Shake..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/46781c13d587e5f951ac1bbb39e57154" - }, - { - "name": "2 McFlurry Oreo (S)", - "price": "158", - "description": "Delicious soft serve meets crumbled oreo cookies, a match made in dessert heaven. Make it double with this combo!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/65f3574f53112e9d263dfa924b1f8fed" - }, - { - "name": "2 Hot Fudge Sundae", - "price": "156", - "description": "A sinful delight, soft serve topped with delicious, gooey hot chocolate fudge. So good you won't be able to stop at one!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f1238db9da73d8ec7a999792f35865d9" - }, - { - "name": "McSpicy Chicken Burger + Fries (M) + Piri Piri Spice Mix", - "price": "295.23", - "description": "Tender and juicy chicken patty coated in spicy, crispy batter topped with a creamy sauce and crispy shredded lettuce will have you craving for more. Served with the spicy piri piri mix and medium fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/e10982204687e18ee6541684365039b8" - }, - { - "name": "McSpicy Paneer Burger + Fries (M) + Piri Piri Spice Mix", - "price": "295.23", - "description": "Rich and filling cottage cheese patty coated in spicy, crispy batter topped with a creamy sauce and crispy shredded lettuce will have you craving for more. Served with the spicy piri piri mix and medium fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/87ce9779f986dcb21ab1fcfe794938d1" - }, - { - "name": "McSpicy Paneer + Cheesy Fries", - "price": "295.23", - "description": "Rich and filling cottage cheese patty coated in spicy, crispy batter topped with a creamy sauce and crispy shredded lettuce will have you craving for more. Served with Cheesy Fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/fcc2fb1635f8e14c69b57126014f0bd5" - }, - { - "name": "Black Forest Mcflurry (M) BOGO", - "price": "139", - "description": "Get 2 Black Forest McFlurry for the price of one!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/15/0319a787-bf68-4cca-a81c-f57d9d993918_9b42d2bc-f1bd-47d4-a5a0-6eb9944ec5cd.png" - }, - { - "name": "New McSaver Chicken Surprise", - "price": "119", - "description": "Enjoy a delicious combo of the new Chicken Surprise Burger with a beverage, now in a delivery friendly reusable bottle.. Contains: Sulphite, Soybeans, Milk, Egg, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/b8c0d92e-86b0-4efa-81c1-66be9b3f846b_ddd80150-1aa4-465c-8453-7f7e7650c6c9.png" - }, - { - "name": "New McSaver Chicken Nuggets (4 Pc)", - "price": "119", - "description": "Enjoy New McSaver Chicken Nuggets (4 Pc).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7bf83367ed61708817caefbc79a3c9eb" - }, - { - "name": "New McSaver McAloo Tikki", - "price": "119", - "description": "Enjoy New McSaver McAloo Tikki.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/ab4c47366f0e51ac0071f705b0f2d93e" - }, - { - "name": "New McSaver Pizza McPuff", - "price": "119", - "description": "Enjoy New McSaver Pizza McPuff.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/45fa406e76418771de26c37e8863fbb3" - }, - { - "name": "Chicken Surprise Burger + McChicken Burger", - "price": "204.76", - "description": "Enjoy the newly launched Chicken Surprise Burger with the iconic McChicken Burger.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/4/f1d9557a-6f86-4eb2-abd5-569f2e865de2_9b71c67a-45f1-41c1-9e1c-b80a934768da.png" - }, - { - "name": "Chicken Surprise Burger + Fries (M)", - "price": "170.47", - "description": "Enjoy the newly launched Chicken Surprise Burger with the iconic Fries (M).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/4/ccceb88b-2015-47a3-857b-21a9641d87ed_ad91cfe4-3727-4f23-a34e-4d09c8330709.png" - }, - { - "name": "Crispy Veggie Burger + Cheesy Fries", - "price": "320", - "description": "Feel the crunch with our newly launched Crispy Veggie Burger with Cheesy Fries.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/d81e09c2-e3c9-4b1e-862f-5b760c429a00_fa2d522d-2987-49c7-b924-965ddd970c0e.png" - }, - { - "name": "Crispy Veggie Burger + McAloo Tikki", - "price": "255.23", - "description": "Feel the crunch with our newly launched Crispy Veggie Burger + McAloo Tikki.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/476ed2c4-89b2-41d2-9903-be339a6a07e5_ff0d9d12-ecbd-43b9-97c9-dd3e3c20a5cb.png" - }, - { - "name": "Crispy Veggie Burger + Piri Piri Fries (M)", - "price": "320", - "description": "Feel the crunch with our newly launched Crispy Veggie Burger with Piri Piri Fries (M).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/abda9650-7216-4a8d-87ce-05342438db59_b501b87f-e104-466e-8af6-d3a9947e76ac.png" - }, - { - "name": "Mc Crispy Chicken Burger + Piri Piri Fries (M)", - "price": "360", - "description": "Feel the crunch with our newly launched McCrispy Chicken Burger with Piri Piri Fries (M).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/02bce46b-37fb-4aaa-a035-4daef4fe5350_1292d3ee-7a34-45a2-adc5-1e5b14b4752a.png" - }, - { - "name": "Mc Crispy Chicken Burger + Cheesy Fries", - "price": "370.47", - "description": "Feel the crunch with our newly launched McCrispy Chicken Burger with Cheesy Fries.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/170e8dd8-d9c1-4b67-ab84-cec5dd4a9e32_6d80a13b-024b-454f-b680-d62c7252cd4d.png" - }, - { - "name": "Chicken Surprise Burger + Cold Coffee", - "price": "266.66", - "description": "Start of your morning energetic and satisfied with our new exciting combo of - Chicken Surprise + Cold Coffee (R).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/10/30/01f2c1c4-e80b-48f0-9323-8e75775e08e3_3dd290a6-ed00-405b-a654-e46ed2854c81.png" - }, - { - "name": "McAloo Tikki Burger + Cold Coffee", - "price": "251.42", - "description": "Start of your morning energetic and satisfied with our new exciting combo of - McAloo Tikki +Cold Coffee (R).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/10/30/78de1f5b-7fa2-4e00-b81d-c6d6392cd9ea_6f6eaf41-6844-4349-b37c-49cb7be685b3.png" - }, - { - "name": "Choco Crunch Cookie + McAloo Tikki Burger", - "price": "144.76", - "description": "A crunchy, chocolatey delight meets the iconic Aloo Tikki Burger,sweet and savory, the perfect duo for your snack-time cravings!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/9229f5fb-a89c-49fc-ac40-45e59fdf69dd_e17fe978-1668-4829-812d-d00a1b46e971.png" - }, - { - "name": "Choco Crunch Cookie + McVeggie Burger", - "price": "223.80", - "description": "A crispy Choco Crunch Cookie and a hearty McVeggie Burger,your perfect balance of sweet indulgence and savory delight in every bite!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/61529326-5b1d-4917-b74f-1b3de9d4e8ef_8dbfb34e-5dd7-4a85-b0dd-3f38be677ff9.png" - }, - { - "name": "Lemon Ice Tea + Choco Crunch Cookie", - "price": "237.14", - "description": "A refreshing Lemon Iced Tea paired with a crunchy Choco Crunch Cookie, sweet, zesty, and perfectly balanced for a delightful treat!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/1b46fafc-e69f-4084-8d73-b47f4291e45b_4b0bdb80-9cd2-4b85-a6b5-13fc7348a3a3.png" - }, - { - "name": "Veg Pizza McPuff + Choco Crunch Cookie", - "price": "139.04", - "description": "A perfect snack duo, savoury, Veg Pizza McPuff paired with a crunchy, chocolatey Choco Crunch Cookie for a delicious treat!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/da76bf5a-f86c-44f7-9587-90ef5bdbe973_cf99d6eb-6cef-4f03-bd64-4e9b94523817.png" - }, - { - "name": "Butter Croissant + Cappuccino..", - "price": "209", - "description": "Buttery croissant paired with a rich, frothy cappuccino.Warm, comforting, and perfectly balanced.A timeless duo for your anytime cravings..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/227a6aa3-f18c-4685-bb09-a76a01ec39a1_f347d737-b634-43d7-9b73-dc99bccde65f.png" - }, - { - "name": "Butter Croissant + Iced Coffee.", - "price": "209", - "description": "Buttery, flaky croissant served with smooth, refreshing iced coffee. A classic combo that's light, crisp, and energizing. Perfect for a quick, satisfying bite..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/be3c8688-f975-4d1e-aad7-9229232bcc69_65c9a679-687d-4e66-9847-eb4d5c0f9825.png" - }, - { - "name": "1 Pc Crispy Fried Chicken", - "price": "108", - "description": "Enjoy the incredibly crunchy and juicy and Crispy Fried Chicken- 1 Pc.. Contains: Egg, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/4e7f77ef46d856205d3e4e4913ffc0e9" - }, - { - "name": "2 Crispy Fried Chicken + 2 McSpicy Fried Chicken + 2 Dips + 2 Coke", - "price": "548.99", - "description": "A combo of crunchy, juicy fried chicken and spicy, juicy McSpicy chicken, with 2 Dips and chilled Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/305180a8-d236-4f9f-9d52-852757dfc0f6_ab67db7f-eb5d-4519-88f5-a704b666db4e.png" - }, - { - "name": "2 Pc Crispy Fried Chicken", - "price": "219", - "description": "Enjoy 2 Pcs of the incredibly crunchy and juicy and Crispy Fried Chicken.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/07ed0b2a381f0a21d07888cf1b1216eb" - }, - { - "name": "1 McSpicy Fried Chk + 1 Crispy Fried Chk + 4 Wings + 2 Coke + 2 Dips", - "price": "532.37", - "description": "Juicy and spicy McSpicy chicken, crispy fried chicken, and wings with 2 Dips and Coke perfect for a flavorful meal..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/14c7ed0d-5164-4411-8c52-2151e93801be_4bf5a209-caed-4b44-91b8-48cd71455f2e.png" - }, - { - "name": "1 Pc McSpicy Fried Chicken", - "price": "114", - "description": "Try the new McSpicy Fried chicken that is juicy, crunchy and spicy to the last bite!. Contains: Sulphite, Soybeans, Peanut, Milk, Egg, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/ceb4e0a0-9bff-48f8-a7da-7e8f53c38b86_c9005198-cc85-420a-81ec-7e0bca0ca8ab.png" - }, - { - "name": "2 Pc McSpicy Chicken Wings", - "price": "93", - "description": "Enjoy the 2 pcs of the New McSpicy Chicken Wings. Spicy and crunchy, perfect for your chicken cravings. Contains: Sulphite, Soybeans, Peanut, Milk, Egg, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6e4918a1cf1113361edba3ed33519ffc" - }, - { - "name": "2 Pc McSpicy Fried Chicken", - "price": "217", - "description": "Try the new McSpicy Fried chicken- 2 pcs that is juicy, crunchy and spicy to the last bite!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/057b4bdb-9b88-4c9a-b54a-fc41d0ea1b5f_c6cd0222-69ae-4f30-b588-9741b82d9195.png" - }, - { - "name": "3 Pc McSpicy Fried Chicken Bucket + 1 Coke", - "price": "380.99", - "description": "Share your love for chicken with 3 pcs of McSpicy Fried Chicken with refreshing coke. The perfect meal for your catchup!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/9832febf-e35a-4575-89c1-dfdefd42bb92_42385d00-210d-4554-ac0f-236268e404a3.png" - }, - { - "name": "4 Pc McSpicy Chicken Wings", - "price": "185", - "description": "Enjoy the 4 pcs of the New McSpicy Chicken Wings. Spicy and crunchy, perfect for your chicken cravings.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7122300975cc9640e84cdc7e7c74e042" - }, - { - "name": "4 Pc McSpicy Fried Chicken Bucket", - "price": "455", - "description": "Share your love for chicken with 4 pcs of McSpicy Fried Chicken..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/5cacf92d-f847-4670-acc2-381f984790d1_60f07bc4-f161-4603-b473-527828f8df08.png" - }, - { - "name": "5 Pc McSpicy Fried Chicken Bucket", - "price": "590", - "description": "Share your love for chicken with 5 pcs of McSpicy Fried Chicken that is spicy to the last bite.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/1e617f1f-2573-475f-a50e-8c9dd02dd451_6ccb66a3-728b-4abb-8a03-4f3eed32da18.png" - }, - { - "name": "12 Pc Feast Chicken Bucket", - "price": "808.56", - "description": "Enjoy 12 pc bucket of 4 Pc McSpicy Fried Chicken + 4 Pc Crispy Fried Chicken+ 4 pc McSpicy Chicken Wings + 2 Medium Cokes + 2 Dips. (Serves 3-4).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/a09f66d7-3ad3-4f8c-9168-6a95849430f5_e360b6f1-659f-42c0-abc0-7f05b95496ce.png" - }, - { - "name": "Chicken Lover's Bucket", - "price": "599.04", - "description": "Enjoy this crunchy combination of 4 Pc McSpicy Chicken Wings + 2 Pc McSpicy Fried Chicken + 2 Pc Crispy Fried Chicken+ 2 Dips + 2 Cokes. A chicken lover's dream come true! (Serves 3-4).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/650f5998-10b0-4c07-bb3b-e68036242f11_c9b5e73c-4bbf-4c04-b918-dd7a32b7973b.png" - }, - { - "name": "4 Chicken Wings + 2 McSpicy Fried Chicken + 2 Coke + 2 Dips", - "price": "528.56", - "description": "Spicy, juicy McSpicy Chicken wings and 2 Pc McSpicy Fried chicken with 2 Dips, paired with 2 chilled cokes.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/42ff8854-3701-4b8c-aace-da723977c2e3_8b62f3cc-32eb-4e84-949e-375045125efc.png" - }, - { - "name": "4 McSpicy Fried Chicken Bucket + 2 Dips + 2 Coke", - "price": "532.37", - "description": "4 pieces of juicy, spicy McSpicy Fried Chicken with 2 Dips and the ultimate refreshment of chilled coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/305180a8-d236-4f9f-9d52-852757dfc0f6_ab67db7f-eb5d-4519-88f5-a704b666db4e.png" - }, - { - "name": "5 McSpicy Fried Chicken Bucket + 2 Dips + 2 Coke", - "price": "566.66", - "description": "5 pieces of juicy, spicy McSpicy Fried chicken with 2 Dips and 2 refreshing Cokes..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/29742264-8fee-4b8f-a8e2-52efb5d4edf9_314d6cbc-b4ea-4d92-92ea-5b7d1df70a9a.png" - }, - { - "name": "8 McSpicy Chicken Wings Bucket + 2 Coke + 1 Dip", - "price": "465.71", - "description": "Juicy, spicy McSpicy Chicken wings with 1 Dip and the ultimate refreshment of chilled coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/ea6a4090-5c1d-4b32-926b-5e1de33574ba_242accc3-d883-4434-9913-fb6c35574c9b.png" - }, - { - "name": "Chicken Surprise Burger with Multi-Millet Bun", - "price": "84.15", - "description": "Try the Chicken Surprise Burger in the new multi-millet bun! Enjoy the same tasty chicken patty you love, now sandwiched between a nutritious multi-millet bun.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/1fb900f6-71bb-4724-a507-830735f555c5_06c06358-bc36-469d-84e3-44e9504bb7d0.png" - }, - { - "name": "McAloo Tikki Burger with Multi-Millet Bun", - "price": "83.91", - "description": "Try your favourite McAloo Tikki Burger in a multi-millet bun! Enjoy the same tasty McAloo Tikki patty you love, now sandwiched between a nutritious millet bun..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/cfd7d779-b04d-43be-947e-a43df33ae119_a0de6b13-c9ac-489a-8416-e9b72ace4542.png" - }, - { - "name": "McChicken Burger with Multi-Millet Bun", - "price": "150.47", - "description": "Make a healthier choice with our McChicken Burger in a multi-millet bun! Same juicy chicken patty, now with a nutritious twist..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/bf0a9899-19a5-4fc9-9898-9e15098bac43_681fbed4-5354-4d5a-9d67-b59d274ea633.png" - }, - { - "name": "McSpicy Chicken Burger with Multi-Millet Bun", - "price": "221.76", - "description": "Feel the heat and feel good too! Try your McSpicy Chicken Burger in nutritious multi-millet bun..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/6eac1f61-cf7b-422a-bc6d-8c55bc1ff842_01e557a3-f5a2-4239-b06b-01ffd46ec154.png" - }, - { - "name": "McSpicy Paneer Burger with Multi-Millet Bun", - "price": "221.76", - "description": "Spice up your meal with a healthier bite! Try your McSpicy Paneer Burger with the nutritious multi-millet bun..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/70eb5956-3666-47f4-bcf8-1f5075207222_fd9e2a06-3f27-47e3-88aa-86895ffaa65e.png" - }, - { - "name": "McVeggie Burger with Multi-Millet Bun", - "price": "158.40", - "description": "Try your favorite McVeggie Burger in a nutritious multi-millet bun! A healthier twist on a classic favorite, with the same tasty veggie patty you love.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/397caee8-4da3-4143-98b1-e4dac155ab3e_0ed975df-7665-4f3a-ae16-8a7eb008afd6.png" - }, - { - "name": "Crispy Veggie Burger Protein Plus (1 Slice)", - "price": "226.71", - "description": "A flavourful patty made with a blend of 7 Premium veggies topped with zesty cocktail sauce now a protein slice to fuel you up, all served between soft premium buns. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/2fcff334-211c-4a2b-bee8-018ce9f10572_71ad5b2a-982d-407a-a833-8f3c1c6bf240.png" - }, - { - "name": "Crispy Veggie Burger Protein Plus (2 Slices)", - "price": "253.43", - "description": "A flavourful patty made with a blend of 7 Premium veggies topped with zesty cocktail sauce now a protein slice to fuel you up, all served between soft premium buns. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/e2c0ecf8-1ca0-447c-a56f-1b8758f7a406_314e4442-4db5-403e-b71c-ffa797622eee.png" - }, - { - "name": "Crispy Veggie Burger Protein Plus + Corn + Coke Zero", - "price": "399", - "description": "A flavourful patty made with a blend of 7 premium veggies, topped with zesty cocktail sauce and now a protein slice to fuel you up, all served between soft premium buns. Paired with corn and Coke Zero..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/d0ac58e8-e601-49e1-abf2-a5456a63f057_a8bb7584-906a-4ce1-891b-6e877f6b4543.png" - }, - { - "name": "McEgg Burger Protein Plus (1 Slice)", - "price": "89.10", - "description": "A steamed egg , spicy habanero sauce and onions and a tasty new protein slice. Simple, satisfying and powered with protein.. Contains: Gluten, Milk, Egg, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/0482392e-9b78-466d-93db-e653c77e0e35_55c40d44-9050-47eb-902e-5639c1423ee7.png" - }, - { - "name": "McEgg Burger Protein Plus (2 Slices)", - "price": "117.80", - "description": "A steamed egg , spicy habanero sauce and onions and a tasty new protein slice. Simple, satisfying and powered with protein.. Contains: Gluten, Milk, Egg, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/a5750dc8-8a1f-48a0-8fa2-83007ee4377c_c8501c16-90f4-4943-80f7-719f519fb918.png" - }, - { - "name": "McAloo Tikki Burger Protein Plus (1 Slice)", - "price": "89.10", - "description": "The OG Burger just got an upgrade with a tasty protein slice.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/6c4f7375-3424-4d01-8425-509381fb3015_25ebe6f6-2f23-4b3f-8dcb-02ea740f91e9.png" - }, - { - "name": "McAloo Tikki Burger Protein Plus (2 Slices)", - "price": "117.80", - "description": "The OG Burger just got an upgrade with a tasty protein slice.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/a95ad198-b6d8-43b0-9960-a1b8d8ceb6f2_12b5df65-ba9d-41ff-b762-88e391b121f8.png" - }, - { - "name": "McAloo Tikki Burger Protein Plus+ Corn + Coke Zero", - "price": "299", - "description": "The OG McAloo Tikki Burger just got an upgrade with a tasty protein slice. Served with buttery corn and a refreshing Coke Zero for a nostalgic yet balanced combo..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/da8beade-c083-4396-b4ff-289b647af81d_b0f6b946-7151-45e3-9fe0-e15a2c46c1d6.png" - }, - { - "name": "McCheese Chicken Burger Protein Plus (1 Slice)", - "price": "297", - "description": "Double the indulgence with sinfully oozing cheesey patty and flame grilled chicken patty , along with chipotle sauce , shredded onion , jalapenos , lettuce and now with a protein slice. Indulgent meets protein power.. Contains: Gluten, Egg, Milk, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/d32ea9af-d134-4ae9-8a1a-8dafddd53cb7_da16f662-ce98-4990-ab8c-566697743730.png" - }, - { - "name": "McCheese Chicken Burger Protein Plus (2 Slices)", - "price": "324.72", - "description": "Double the indulgence with sinfully oozing cheesey patty and flame grilled chicken patty , along ith chipotle sauce , shredded onion , jalapenos , lettuce and now with a protein slice. Indulgent meets protein power.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/e2b28b79-63df-43bb-8c41-240a076e0a3a_49dbc6df-3f9d-448c-bd94-377ebcc3a335.png" - }, - { - "name": "McCheese Chicken Burger Protein Plus + 4 Pc Chicken Nuggets+ Coke Zero", - "price": "449", - "description": "Double the indulgence with sinfully oozing cheesy patty and flame-grilled chicken patty, chipotle sauce, shredded onion, jalapenos, lettuce, and now a protein slice. Served with 4 Pc Chicken nuggets and Coke Zero. Indulgent meets protein power..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/f9aae1ec-6313-4684-aeb3-50de503fcf23_f5b200ed-60fa-42f6-928f-6b95327da3bb.png" - }, - { - "name": "McChicken Burger Protein Plus (1 Slice)", - "price": "185.13", - "description": "The classic McChicken you love, made more wholesome with a protein slice. Soft, savoury, and now protein rich.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/20fd3d59-0003-4e80-ad71-79ca8ae4d050_8bf1219b-a67d-4518-a115-167fa10bc440.png" - }, - { - "name": "McChicken Burger Protein Plus + 4 Pc Chicken Nuggets + Coke Zero", - "price": "349", - "description": "The classic McChicken you love, made more wholesome with a protein slice. Soft, savoury, and now protein-rich. Comes with 4 crispy Chicken nuggets and a chilled Coke Zero..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/afdcdd59-c5f0-44f1-9014-b28455ce0ecf_059ea761-4f24-4e04-b152-cdb82a04b366.png" - }, - { - "name": "McChicken Protein Burger Plus (2 Slices)", - "price": "212.84", - "description": "The classic McChicken you love, made more wholesome with a protein slice.Soft, savoury, and now protein-rich.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/1e1f281e-8378-4993-a7c1-d6f319a7bd17_768a7827-1fe8-43a6-ba35-a0ccb5a95ef5.png" - }, - { - "name": "McCrispy Chicken Burger Protein Plus (1 Slice)", - "price": "246.51", - "description": "A Crunchy , golden chicken thigh fillet , topped with fresh lettuce and creamy pepper mayo now also with a hearty protein slice all nestled between soft toasted premium buns.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/4f908fa6-2fa5-4367-9da9-9039cb5e72e0_c8570551-57ae-441a-81c9-64335392ac3b.png" - }, - { - "name": "McCrispy Chicken Burger Protein Plus (2 Slices)", - "price": "276.20", - "description": "A Crunchy , golden chicken thigh fillet , topped with fresh lettuce and creamy pepper mayo now also with a hearty protein slice all nestled between soft toasted premium buns.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/fc4e713d-5ee0-421e-9690-7c8b2c587e5a_2b710d54-3ae7-46fe-990c-140b34494dee.png" - }, - { - "name": "McCrispy Chicken Burger Protein Plus + 4 Pc Chicken Nugget + Coke Zero", - "price": "419", - "description": "A crunchy, golden chicken thigh fillet topped with fresh lettuce and creamy pepper mayo, now also with a hearty protein slice, all nestled between soft toasted premium buns. Comes with 4-piece nuggets and Coke Zero..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/fdb3d9b1-2262-4b18-9264-9bd00b3213ba_20858442-9b85-4728-b7b2-07d72349782e.png" - }, - { - "name": "McEgg Burger Protein Plus + 4 Pc Chicken Nuggets + Coke Zero", - "price": "299", - "description": "A steamed egg, spicy habanero sauce, onions, and a tasty new protein slice. Simple, satisfying, and powered with protein. Served with 4-piece chicken nuggets and Coke Zero..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/47092b74-fac3-48a2-9abc-1d43b975199f_d07e626a-6cb9-4664-8517-4ce7ba71b234.png" - }, - { - "name": "McSpicy Chicken Burger Protein Plus (1 Slice)", - "price": "225.72", - "description": "Indulge in our signature tender chicken patty coated in spicy crispy batter , topped with creamy sauce ,crispy lettuce and now with a new protein slice .. Contains: Gluten, Milk, Egg, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/79035335-855d-4b89-93e4-c063258aaf64_22e9ada8-6768-481d-83d3-7223bc119bce.png" - }, - { - "name": "McSpicy Chicken Burger Protein Plus (2 Slices)", - "price": "252.44", - "description": "Indulge in our signature tender chicken patty coated in spicy crispy batter , topped with creamy sauce ,crispy lettuce and now with a new protein slice .. Contains: Gluten, Egg, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/36e2e35c-86d5-4b0d-b573-325c7b4e5fd9_4bc54386-d43e-4d28-baca-d0da22bc3668.png" - }, - { - "name": "McSpicy Chicken Burger Protein Plus + 4 Pc Chicken Nuggets + Coke Zero", - "price": "399", - "description": "Indulge in our signature tender chicken patty coated in spicy crispy batter , topped with creamy sauce ,crispy lettuce and now with a new protein slice Served with 4-piece chicken nuggets and Coke Zero..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/a81bd599-05ad-4453-9ada-499d9cf45b29_1096d08c-d519-4133-babb-2f54a9b6e5f0.png" - }, - { - "name": "McSpicy Paneer Burger Protein Plus (1 Slice)", - "price": "226.71", - "description": "Indulge in rich and filling spicy paneer patty served with creamy sauce, crispy lettuce and now with a new protein slice. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/3ca8296e-9885-4df6-8621-5349298a150f_a2ad6b56-b6ae-4e4b-986c-abe75e052d75.png" - }, - { - "name": "McSpicy Paneer Burger Protein Plus (2 Slices)", - "price": "253.43", - "description": "Indulge in rich and filling spicy paneer patty served with creamy sauce, crispy lettuce and now with a new protein slice. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/c2561720-1064-4043-a143-3c9ae893e481_b09ff597-1b69-4098-ba04-6ab3f3b09c6f.png" - }, - { - "name": "McSpicy Paneer Burger Protein Plus + Corn + Coke Zero", - "price": "399", - "description": "Indulge in rich and filling spicy paneer patty served with creamy sauce, crispy lettuce and now with a new protein slice. Served with Corn and Coke Zero..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/d5c3f802-ad34-4086-ae66-731406a82c99_4b87555b-5e27-40da-b7f1-fd557212e344.png" - }, - { - "name": "McSpicy Premium Burger Veg Protein Plus (2 Slices)", - "price": "303.93", - "description": "A wholesome spicy paneer patty, lettuce topped with jalapenos and cheese slice and now with a protein-packed slice for that extra boost , spicy cocktail sauce and cheese sauce.. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/05949fc3-08a4-4e47-8fd1-16abc12a952c_0b80fed2-213d-4945-9ea7-38c818cdb6ef.png" - }, - { - "name": "McSpicy Premium Chicken Burger Protein Plus (1 Slice)", - "price": "287.10", - "description": "A wholesome spicy chicken patty lettuce topped with jalapenos and cheese slice plus an added protein slice , spicy cocktail sauce and cheese sauce.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/690ecde3-4611-4323-8436-a1dadfe2eb7b_fe9b5993-f422-4bae-8211-9715dd1d51d8.png" - }, - { - "name": "McSpicy Premium Chicken Burger Protein Plus (2 Slices)", - "price": "315.80", - "description": "A wholesome spicy chicken patty lettuce topped with jalapenos and cheese slice plus an added protein slice , spicy cocktail sauce and cheese sauce.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/80bf28f3-b317-4846-b678-95cfc857331a_d247479e-0bcb-4f5d-90da-b214d30e70a3.png" - }, - { - "name": "McSpicy Premium Chicken Protein Plus + 4 Pc Chicken Nugget + Coke Zero", - "price": "479", - "description": "A wholesome spicy chicken patty, lettuce topped with jalapenos and cheese slice, plus an added protein slice. Comes with spicy cocktail sauce and cheese sauce,served with 4 Pc crispy chicken nuggets and a Coke Zero..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/a512848a-d34d-44cf-a0db-f7e84781c63d_6d8fa5df-a1c6-42e6-b88b-929c59c4f50c.png" - }, - { - "name": "McSpicy Premium Veg Burger Protein Plus (1 Slice)", - "price": "276.20", - "description": "A wholesome spicy paneer patty, lettuce topped with jalapenos and cheese slice and now with a protein packed slice for that extra boost , spicy cocktail sauce and cheese sauce.. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/8768ff6a-ea10-4260-91e6-018695820d79_2399dc04-b793-4336-b2e4-a6240cdc5f78.png" - }, - { - "name": "McSpicy Premium Veg Burger Protein Plus + Corn + Coke Zero", - "price": "449", - "description": "A wholesome spicy paneer patty, lettuce topped with jalapenos and cheese slice, now with a protein-packed slice for that extra boost. Comes with spicy cocktail sauce, cheese sauce, buttery corn, and a chilled Coke Zero..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/d183fbeb-62dc-4b3d-936f-99ddc9dc5466_ce66dc12-da9f-479d-bd4a-c77fdb3dae2c.png" - }, - { - "name": "McVeggie Burger Protein Plus (1 Slice)", - "price": "185.13", - "description": "The classic McVeggie you love, made more wholesome with a protein slice.Soft, savoury, and now protein-rich.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/99aedaa8-836b-4893-b7f9-24b3303b6c94_781b8449-5bc9-42e5-9747-00bcec474deb.png" - }, - { - "name": "McVeggie Burger Protein Plus (2 Slices)", - "price": "212.88", - "description": "The classic McVeggie you love, made more wholesome with a protein slice.Soft, savoury, and now protein rich.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/630f74dd-841a-4641-bf7f-c23cd9e4e0d5_0fd861eb-43f6-46c1-9bfe-f8043c2aae6a.png" - }, - { - "name": "McVeggie Burger Protein Plus + Corn + Coke Zero", - "price": "339", - "description": "The classic McVeggie you love, made more wholesome with a protein slice. Soft, savoury, and now protein rich. Served with sweet corn and Coke Zero for a balanced meal..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/f448d30d-9bbe-4659-bbb4-14b22d93b0ae_0a188512-5619-4d62-980f-7db8e45f6ebe.png" - }, - { - "name": "Big Group Party Combo 6 Veg", - "price": "760.95", - "description": "Enjoy a Big Group Party Combo of McAloo + McVeggie + McSpicy Paneer + Mexican McAloo + Corn and Cheese + Crispy Veggie burger.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/427a977d-1e14-4fe9-a7ac-09a71f578514_e3089568-134d-45fe-b97e-59d756892f15.png" - }, - { - "name": "Big Group Party Combo for 6 Non- Veg", - "price": "856.19", - "description": "Enjoy a Big Group Party Combo of Surprise Chicken + McChicken + McSpicy Chicken + Grilled Chicken + McSpicy Premium + Mc Crispy Chicken Burger .", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/e1a453be-6ae4-4e82-8957-c43756d2d72a_d7cac204-1cb6-4b6d-addf-c0324435cc58.png" - }, - { - "name": "Big Group Party Combo1 for 4 Non- Veg", - "price": "475.23", - "description": "Save on your favourite Big Group Party Combo - Surprise Chicken + McChicken + McSpicy Chicken + Grilled Chicken Burger.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/d955dfa6-3a2c-46d2-b59c-e9e59b0939c3_1417a17c-bb7e-4e24-8931-3950cac2f074.png" - }, - { - "name": "Big Group Party Combo1 for 4 Veg", - "price": "475.23", - "description": "Get the best value in your Combo for 4 Save big on your favourite Big Group Party Combo-McAloo + McVeggie + McSpicy Paneer + Corn and Cheese Burger.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/5c723cbf-e7e2-48cc-a8fe-6f4e0eeea73d_899a4709-e9fb-4d9f-a997-973027fc0e7d.png" - }, - { - "name": "Big Group Party Combo2 for 4 Non-Veg", - "price": "522.85", - "description": "Your favorite party combo of 2 McChicken + 2 McSpicy Chicken Burger.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/b648ba7b-e82d-4d37-ab47-736fdb12cd90_19ef381a-3cc6-4b50-9e69-911f3656987f.png" - }, - { - "name": "2 Crispy Veggie Burger + Fries (L) + 2 Coke", - "price": "635.23", - "description": "Feel the crunch with Burger Combos for 2: 2 Crispy Veggie Burger + Fries (L)+ 2 Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/ce6551d8-829a-4dde-9131-cbf7818a6f26_a5b69cee-6377-4295-bbee-49725693c45d.png" - }, - { - "name": "Big Group Party Combo2 for 4 Veg", - "price": "522.85", - "description": "Your favorite party combo of 2 McVeggie + 2 McSpicy Paneer Burger.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/245c7235-500d-4212-84bd-f60f9aaf27be_73b24a53-d304-4db0-9336-061760ca17a9.png" - }, - { - "name": "2 Crispy Veggie Burger + 2 Fries (M) + Veg Pizza McPuff", - "price": "604.76", - "description": "Feel the crunch with Burger Combos for 2: 2 Crispy Veggie Burger + 2 Fries (M)+ Veg Pizza McPuff.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/ca2023e7-41e0-4233-a92c-c2bc52ef8ee5_33a0ea24-50ee-4a9b-b917-bbc2e6b5f37b.png" - }, - { - "name": "Crispy Veggie Burger + McVeggie Burger + Fries (M)", - "price": "424.76", - "description": "Feel the crunch with Crispy Veggie Burger+ McVeggie + Fries (M).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/c6dd67c1-0af9-4333-a4b5-3501e0ce4579_2fee69c9-f349-46c5-bdd9-8965b9f76687.png" - }, - { - "name": "Burger Combo for 2: McAloo Tikki", - "price": "364.76", - "description": "Stay home, stay safe and share a combo- 2 McAloo Tikki Burgers + 2 Fries (L).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/ea7ba594c7d77cb752de9a730fbcb3bf" - }, - { - "name": "6 Pc Chicken Nuggets + McChicken Burger + Coke", - "price": "375.22", - "description": "Tender and juicy chicken patty cooked to perfection, with creamy mayonnaise and crunchy lettuce adding flavour to each bite. Served with 6 Pc Nuggets and Coke..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/934194567f9c231dc46dccf2d4e6d415" - }, - { - "name": "Burger Combo for 2: McSpicy Chicken + McChicken", - "price": "464.76", - "description": "Flat 15% Off on McSpicy Chicken Burger + McChicken Burger + Fries (M).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/10ada13e-5724-487f-8ab6-fd07005859ad_a57d565d-0dfe-4424-bc5b-77b16143ad63.png" - }, - { - "name": "Burger Combo for 2: Corn & Cheese + McVeggie", - "price": "404.76", - "description": "Flat 15% Off on Corn & Cheese Burger +McVeggie Burger+Fries (M).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/08e9bc73-6774-41cf-96bb-ca817c4e23d3_e00ec89e-86f8-4531-956a-646082dc294c.png" - }, - { - "name": "Burger Combo for 2: McSpicy Chicken Burger with Pizza McPuff", - "price": "535.23", - "description": "Save big on your favourite sharing combo- 2 McSpicy Chicken Burger + 2 Fries (M) + Veg Pizza McPuff.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/e62aea3ba1cd5585a76004f59cd991e5" - }, - { - "name": "Burger Combo for 2: McSpicy Paneer + McAloo Tikki with Pizza McPuff", - "price": "427.61", - "description": "Get the best value in your meal for 2. Save big on your favourite sharing meal - McSpicy Paneer Burger + 2 Fries (M) + McAloo Tikki Burger + Veg Pizza McPuff.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/0ea8a2fddbbc17bc6239a9104963a3e8" - }, - { - "name": "Burger Combo for 2: McChicken Burger", - "price": "464.75", - "description": "Save big on your favourite sharing combo - 2 McChicken Burger + Fries (L) + 2 Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/7/177036e5-4afe-4076-b9cd-d7031f60ffe8_97687ef4-e242-4625-9b3c-398c60b8ddf2.png" - }, - { - "name": "Burger Combo for 2: McSpicy Chicken Burger", - "price": "548.56", - "description": "Save big on your favourite sharing combo - 2 McSpicy Chicken Burger + Fries (L) + 2 Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/98afaf26d81b15bec74cc356fe60cc13" - }, - { - "name": "Burger Combo for 2: McVeggie Burger", - "price": "424.75", - "description": "Save big on your favourite sharing combo - 2 McVeggie Burger + Fries (L) + 2 Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/09b3cb6130cfae15d486223c313fb6c6" - }, - { - "name": "2 Chicken Maharaja Mac Burger + 2 Coke + Fries (L) + McFlurry Oreo (M)", - "price": "670.47", - "description": "Enjoy 2 of the tallest burgers innovated by us. Created with chunky juicy grilled chicken patty paired along with fresh ingredients like jalapeno, onion, slice of cheese, tomatoes & crunchy lettuce dressed with the classical Habanero sauce. Served with Coke, Large Fries and a medium McFlurry Oreo.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/65c9c9b82c4d1f77a05dc4d89c9ead1d" - }, - { - "name": "Burger Combo for 2: Corn & Cheese Burger", - "price": "464.75", - "description": "Save big on your favourite sharing combo - 2 Corn and Cheese Burger + Fries (L) + 2 Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/847b562672e71c2352d92b797c0b0a4e" - }, - { - "name": "Burger Combo for 2: Grilled Chicken & Cheese", - "price": "495.23", - "description": "Save big on your favourite sharing combo - 2 Grilled Chicken and Cheese Burger + Fries (L) + 2 Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/1166a8baa3066342affb829ef0c428dd" - }, - { - "name": "2 Mc Crispy Chicken Burger + Fries (L) + 2 Coke", - "price": "724.75", - "description": "Feel the crunch with our Burger Combos for 2 : 2 McCrispy Chicken Burger + Fries (L)+ 2 Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/b4fda3b0-82c9-4d97-ab9c-c804b7d7893a_a14992d5-49cb-4035-827d-a43e40752840.png" - }, - { - "name": "Mc Crispy Chicken Burger + McChicken Burger + Fries (M)", - "price": "430.47", - "description": "Feel the crunch with McCrispy Chicken Burger+ McChicken + Fries (M).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/eaeeaf30-7bb6-4bef-9b78-643533a4520c_e54bb308-0447-44c2-9aaf-f36ce25f7939.png" - }, - { - "name": "Mc Crispy Chicken Burger + McSpicy Chicken Wings - 2 pc + Coke (M)", - "price": "415.23", - "description": "Feel the crunch with McCrispy Chicken Burger+ McSpicy Chicken Wings - 2 pc + Coke (M).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/f8669731-b3e5-434d-8b45-269b75555b9b_3dfd8e01-5472-40db-a266-bb055036531b.png" - }, - { - "name": "McChicken Double Patty Burger Combo", - "price": "352.99", - "description": "Your favorite McChicken Burger double pattu burger + Fries (M) + Drink of your choice in a new, delivery friendly, resuable bottle..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/e8a8f43ee29a3b97d2fac37e89648eac" - }, - { - "name": "McSpicy Chicken Double Patty Burger combo", - "price": "418.99", - "description": "Your favorite McSpicy Chicken double patty Burger + Fries (M) + Drink of your choice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/91ed96b67df6e630a6830fc2e857b5b1" - }, - { - "name": "McVeggie Burger Happy Meal", - "price": "298.42", - "description": "Enjoy a combo of McVeggie Burger + Sweet Corn + B Natural Mixed Fruit Beverage + Book.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/15e7fc6b-f645-4ed9-a391-1e1edc452f9f_47ad6460-6af4-43b4-8365-35bb0b3fc078.png" - }, - { - "name": "McChicken Burger Happy Meal", - "price": "321.42", - "description": "Enjoy a combo of McChicken Burger + Sweet Corn + B Natural Mixed Fruit Beverage + Book.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/363ca5bf-bb04-4847-bc4e-5330a27d3874_8b5e46ed-61ba-4e8f-b291-2955af07eba0.png" - }, - { - "name": "McAloo Tikki Burger Happy Meal", - "price": "205.42", - "description": "Enjoy a combo of McAloo Tikki Burger + Sweet Corn + B Natural Mixed Fruit Beverage + Book.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/a9f2acb6-3fd3-4691-beb0-61439337f907_6733a4ee-3eb5-46da-9ab7-1b6943f68be7.png" - }, - { - "name": "Big Spicy Paneer Wrap Combo", - "price": "360.99", - "description": "Your favorite Big Spicy Paneer Wrap + Fries (M) + Drink of your choice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/22/116148a2-7f56-44af-bdd0-6bbb46f48baa_355bbd20-86d4-49b8-b3a9-d290772a9676.png" - }, - { - "name": "9 Pc Chicken Nuggets Combo", - "price": "388.98", - "description": "Enjoy your favorite Chicken McNuggets + Fries (M) + Drink of your choice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/4c56f086e500afe6b2025f9c46846e12" - }, - { - "name": "Mexican McAloo Tikki Burger Combo", - "price": "223.99", - "description": "Enjoy a delicious combo of Mexican McAloo Tikki Burger + Fries (M) + Beverage of your choice in a new, delivery friendly, reusable bottle..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/0140c49c7274cdb6af08053af1e6cc20" - }, - { - "name": "McEgg Burger Combo", - "price": "230.99", - "description": "Enjoy a combo of McEgg + Fries (M) + Drink of your Choice . Order now to experience a customizable, delicious meal..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/4f474c833930fa31d08ad2feed3414d8" - }, - { - "name": "McChicken Burger Combo", - "price": "314.99", - "description": "Your favorite McChicken Burger + Fries (M) + Drink of your choice in a new, delivery friendly, resuable bottle..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/b943fe56-212d-4366-a085-4e24d3532b30_3776df33-e18d-46c9-a566-c697897f1d16.png" - }, - { - "name": "McSpicy Chicken Burger Combo", - "price": "363.99", - "description": "Your favorite McSpicy Chicken Burger + Fries (M) + Drink of your choice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/214189a0-fd53-4f83-9d18-c3e53f2c017a_70748bf8-e587-47d6-886c-9b7ac57e84b3.png" - }, - { - "name": "McSpicy Paneer Burger Combo", - "price": "344.99", - "description": "Enjoy your favourite McSpicy Paneer Burger + Fries (M) + Drink of your Choice . Order now to experience a customizable, delicious combo.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/70fc7aa0-7f3b-418e-8ccd-5947b5f1aacd_055bbe77-fbe7-4200-84d3-4349475eb298.png" - }, - { - "name": "Veg Maharaja Mac Burger Combo", - "price": "398.99", - "description": "Enjoy a double decker Veg Maharaja Mac+ Fries (M) + Drink of your Choice . Order now to experience a customizable, delicious meal..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/d064fb5e-fb2e-4e1d-9515-18f26489f5b1_f6f49dba-2ae7-4ebf-8777-548c5ad4d799.png" - }, - { - "name": "McVeggie Burger Combo", - "price": "308.99", - "description": "Enjoy a combo of McVeggie + Fries (M) + Drink of your Choice in a new, delivery friendly, resuable bottle..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/5c9942c5-bc9d-4637-a312-cf51fe1d7aa8_e7e13473-9424-4eb8-87c3-368cfd084a2f.png" - }, - { - "name": "Big Spicy Chicken Wrap Combo", - "price": "379.99", - "description": "Your favorite Big Spicy Chicken Wrap + Fries (M) + Drink of your choice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/22/f631f834-aec2-410d-ab9d-7636cd53d7a0_8801a3c7-ad83-4f7c-bcb0-76101fadde91.png" - }, - { - "name": "Chicken Maharaja Mac Burger Combo", - "price": "398.99", - "description": "Enjoy a double decker Chicken Maharaja Mac + Fries (M) + Drink of your Choice . Order now to experience a customizable, delicious meal..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/ead090f2-5f80-4159-a335-d32658bcfc7c_8bcc5cd9-b22a-4f5d-8cde-0a2372c985c8.png" - }, - { - "name": "Grilled Cheese and Chicken Burger Combo", - "price": "310.47", - "description": "Enjoy a combo of Grilled Chicken & Cheese Burger + Fries (M) + Coke . Order now to experience a customizable, delicious meal..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/f6813404-54a4-492b-a03a-cf32d00ee1ae_c98c4761-69eb-4503-bde3-dffaafd43b15.png" - }, - { - "name": "Corn & cheese Burger Combo", - "price": "310.47", - "description": "Enjoy a combo of Corn & Cheese Burger + Fries (M) + Coke . Order now to experience a customizable, delicious meal..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/c8839f22-e525-44bb-8a50-8ee7cffecd26_cf4c0bcf-c1b9-4c0e-9109-03eb86abf4dd.png" - }, - { - "name": "Birthday Party Package - McChicken", - "price": "2169.14", - "description": "5 McChicken Burger + 5 Sweet Corn + 5 B Natural Mixed Fruit Beverage + 5 Soft Serve (M) + Book.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/15/37e0bc09-3e46-41d6-8146-66d6962ae2ab_0c91e294-cad2-4177-a2a9-86d6523bb811.png" - }, - { - "name": "Birthday Party Package - McVeggie", - "price": "2169.14", - "description": "5 McVeggie Burger + 5 Sweet Corn + 5 B Natural Mixed Fruit Beverage + 5 Soft Serve (M) + Book.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/15/0be6390f-e023-47f8-8bf4-9bb16d4995ce_2eceb8c2-6e90-47d7-97d6-93960391e668.png" - }, - { - "name": "McEgg Burger Happy Meal", - "price": "231.42", - "description": "Enjoy a combo of McEgg Burger + Sweet Corn + B Natural Mixed Fruit Beverage + Book.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/ada14e04-680d-44ef-bb05-180d0cc26ebc_b7bda720-9fcd-4bf4-97eb-bfb5a98e0239.png" - }, - { - "name": "McCheese Burger Veg Combo", - "price": "388.99", - "description": "Enjoy a deliciously filling meal of McCheese Veg Burger + Fries (M) + Beverage of your Choice in a delivery friendly, reusable bottle..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/22b25dd0-80ed-426b-a323-82c6b947612d_df827568-54f9-4329-af60-1fff38d105e9.png" - }, - { - "name": "McSpicy Premium Burger Chicken Combo", - "price": "398.99", - "description": "A deliciously filling meal of McSpicy Premium Chicken Burger + Fries (M) + Drink of your choice.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/28550356-fea3-47ce-bfa8-11222c78958a_8975c012-a121-42c3-92e7-719644761d83.png" - }, - { - "name": "McSpicy Premium Burger Veg Combo", - "price": "384.99", - "description": "A deliciously filling meal of McSpicy Premium Veg Burger + Fries (M) + Drink of your choice.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/14ce2cab-913c-4916-b98a-df2e5ea838ff_6d6208ca-4b76-4b87-a3fe-72c2db1081d8.png" - }, - { - "name": "McCheese Burger Chicken Combo", - "price": "388.99", - "description": "Enjoy a deliciously filling meal of McCheese Chicken Burger + Fries (M) + Beverage of your Choice in a delivery friendly, reusable bottle..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/b32eb9b3-7873-4f00-b601-0c92dd5a71ef_ec318a73-0e41-4277-8606-dc5c33b04533.png" - }, - { - "name": "McAloo Tikki Burger Combo", - "price": "204.99", - "description": "Enjoy a delicious combo of McAloo Tikki Burger + Fries (M) + Beverage of your choice in a new, delivery friendly, reusable bottle..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/b03a3ad7212fca0da40e90eed372ced9" - }, - { - "name": "McCheese Burger Veg Combo with Corn", - "price": "415.99", - "description": "Enjoy a combo of McCheese Burger Veg, Classic corn, McFlurry Oreo (Small) with a beverage of your choice in a delivery friendly, resuable bottle..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/9068eeb6-c774-4354-8c18-bf2ddcc94d10_631c39d1-d78c-4275-a14f-60670ce5aee4.png" - }, - { - "name": "2 Pc Chicken Nuggets Happy Meal", - "price": "211.42", - "description": "Enjoy a combo of 2 Pc Chicken Nuggets + Sweet Corn+ B Natural Mixed Fruit Beverage + Book.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/ac8a1d83-59c4-438f-bccb-edd601dacbf5_872c3881-77a1-43f7-82ca-2a929886045d.png" - }, - { - "name": "4 Pc Chicken Nuggets Happy Meal", - "price": "259.42", - "description": "Enjoy a combo of 4 Pc Chicken Nuggets + Sweet Corn+ B Natural Mixed Fruit Beverage + Book.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/fab04bb4-4f67-459a-85ef-5e9ae7861c88_f6761c27-26b3-4456-8abb-8f6c54274b34.png" - }, - { - "name": "Chicken McNuggets 6 Pcs Combo", - "price": "350.99", - "description": "Enjoy your favorite Chicken McNuggets + Fries (M) + Drink of your choice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6e7f9411ed67fe8d8873734af1e8d4e9" - }, - { - "name": "Chicken Surprise Burger + 4 Pc Chicken McNuggets + Coke", - "price": "259.99", - "description": "Enjoy the newly launched Chicken Surprise Burger with 4 Pc Chicken McNuggets and Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/4/6c0288a4-943a-4bb6-a810-b14877c0ea8f_f56187da-4ae4-4a88-b1a5-e48e28d78087.png" - }, - { - "name": "Chicken Surprise Burger Combo", - "price": "238.09", - "description": "Chicken Surprise Burger + Fries (M) + Drink of your choice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/cb9833dd-5983-4445-bb1b-8d0e70b6c930_8928ae23-ddae-4d9a-abc5-e887d7d7868e.png" - }, - { - "name": "Crispy Veggie Burger Meal (M)", - "price": "326.99", - "description": "A flavorful patty with 7 premium veggies, zesty cocktail sauce, and soft buns, paired with crispy fries (M) and a refreshing Coke (M). A perfectly satisfying and full-flavored meal!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/eb4650ba-e950-4953-b1a7-c03a691ac0d2_6ca20c30-4c62-462a-b46d-9d1f21786b49.png" - }, - { - "name": "Mc Crispy Chicken Burger Meal (M)", - "price": "366.99", - "description": "A crunchy, golden chicken thigh fillet with fresh lettuce and creamy pepper mayo between soft, toasted premium buns, served with crispy fries (M) and a refreshing Coke (M). A perfectly satisfying and full-flavored meal!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/eedf0540-f558-4945-b996-994b1cd58048_d9af8cc8-384e-48c3-ab59-2facadfe574a.png" - }, - { - "name": "Choco Crunch Cookie + McAloo Tikki Burger + Lemon Ice Tea", - "price": "284.76", - "description": "Indulge in the perfect combo,crispy Choco Crunch Cookie, classic Aloo Tikki Burger, and refreshing Lemon Iced Tea. A delicious treat for your cravings, delivered fresh to your doorstep!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/b8ed02df-fb8f-4406-a443-398731aa9ef3_dafd8af6-e740-4987-8f4a-7bdc51dda4d8.png" - }, - { - "name": "Veg Pizza McPuff + Choco Crunch Cookie + Americano", - "price": "284.76", - "description": "A delightful trio, savoury Veg Pizza McPuff, crunchy Choco Crunch Cookie, and bold Americano, perfect for a satisfying snack break!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/9d81dad2-06a4-4402-9b07-2ed418963e16_f256697b-a444-4dfa-a9ff-d6448bb67b4b.png" - }, - { - "name": "Big Yummy Cheese Meal (M)", - "price": "424.76", - "description": "Double the indulgence, double the flavor: our Big Yummy Cheese Burger meal layers a spicy paneer patty and Cheese patty with crisp lettuce and smoky chipotle sauce, served with fries (M) and a beverage of your choice.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/cb892b9e-48b5-4127-b84f-b5e395961ddf_755bf2c0-e30a-4185-81e7-c35cbd07f483.png" - }, - { - "name": "Big Yummy Chicken Meal (M)", - "price": "424.76", - "description": "Indulge in double the delight: our Big Yummy Chicken Burger meal pairs the tender grilled chicken patty and Crispy chicken patty with crisp lettuce, jalapeños, and bold chipotle sauce, served with fries (M) and a beverage of your choice ..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/93313e0c-da3b-4490-839e-fd86b84c2644_b0076aa7-ef47-4f13-b72a-40f64744db95.png" - }, - { - "name": "McSpicy Chicken Double Patty Burger", - "price": "278.19", - "description": "Indulge in our signature tender double chicken patty, coated in spicy, crispy batter, topped with creamy sauce, and crispy lettuce.. Contains: Soybeans, Milk, Egg, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/314b5b5786f73746de4880602723a913" - }, - { - "name": "McChicken Double Patty Burger", - "price": "173.24", - "description": "Enjoy the classic, tender double chicken patty with creamy mayonnaise and lettuce in every bite. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/af88f46a82ef5e6a0feece86c349bb00" - }, - { - "name": "McVeggie Double Patty Burger", - "price": "186.12", - "description": "Savour your favorite spiced double veggie patty, lettuce, mayo, between toasted sesame buns in every bite. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/2d5062832f4d36c90e7dfe61ef48e85a" - }, - { - "name": "Mexican McAloo Tikki Double Patty Burger", - "price": "93.05", - "description": "A fusion of International taste combined with your favourite aloo tikki now with two patties. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/cda0e2d51420a95fad28ad728914b6de" - }, - { - "name": "McAloo Tikki Double Patty Burger", - "price": "88.11", - "description": "The World's favourite Indian burger! A crispy double Aloo patty, tomato mayo sauce & onions. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/ef569f74786e6344883a1decdd193229" - }, - { - "name": "McAloo Tikki Burger", - "price": "69.30", - "description": "The World's favourite Indian burger! A crispy Aloo patty, tomato mayo sauce & onions. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/b13811eeee71e578bc6ca89eca0ec87f" - }, - { - "name": "Big Spicy Paneer Wrap", - "price": "239.58", - "description": "Rich & filling cottage cheese patty coated in spicy crispy batter, topped with tom mayo sauce wrapped with lettuce, onions, tomatoes & cheese.. Contains: Sulphite, Soybeans, Peanut, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/22/198c3d14-3ce8-4105-8280-21577c26e944_779c4353-2f85-4515-94d2-208e90b830eb.png" - }, - { - "name": "McSpicy Chicken Burger", - "price": "226.71", - "description": "Indulge in our signature tender chicken patty, coated in spicy, crispy batter, topped with creamy sauce, and crispy lettuce.. Contains: Soybeans, Egg, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/dcdb436c-7b9f-4667-9b73-b8fa3215d7e2_9730340a-661b-49f4-a7d9-a8a89ffe988f.png" - }, - { - "name": "McSpicy Paneer Burger", - "price": "225.72", - "description": "Indulge in rich & filling spicy paneer patty served with creamy sauce, and crispy lettuce—irresistibly satisfying!. Contains: Sulphite, Soybeans, Peanut, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/fb912d21-ad9d-4332-b7cf-8f65d69e2c47_1fa5998c-b486-449a-9a88-2e61cf92ff77.png" - }, - { - "name": "Mexican McAloo Tikki Burger", - "price": "75.42", - "description": "Your favourite McAloo Tikki with a fusion spin with a Chipotle sauce & onions. Contains: Sulphite, Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/167aeccf27bab14940fa646c8328b1b4" - }, - { - "name": "McVeggie Burger", - "price": "153.44", - "description": "Savour your favorite spiced veggie patty, lettuce, mayo, between toasted sesame buns in every bite. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/2cf63c01-fef1-49b6-af70-d028bc79be7b_bfe88b73-33a9-489c-97f3-fb24631de1fc.png" - }, - { - "name": "McEgg Burger", - "price": "69.30", - "description": "A steamed egg, spicy Habanero sauce, & onions on toasted buns, a protein packed delight!. Contains: Soybeans, Milk, Egg, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/265c57f68b1a52f1cc4b63acf082d611" - }, - { - "name": "Veg Maharaja Mac Burger", - "price": "246.51", - "description": "Savor our filling 11 layer burger! Double the indulgence with 2 corn & cheese patties, along with jalapeños, onion, cheese, tomatoes, lettuce, and spicy Cocktail sauce. . Contains: Sulphite, Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/06354d09-be1b-406c-86b5-49dc9b5062d1_2f80f39e-c951-4ca6-8fca-a243a18c3448.png" - }, - { - "name": "McChicken Burger", - "price": "150.47", - "description": "Enjoy the classic, tender chicken patty with creamy mayonnaise and lettuce in every bite. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/c093ba63-c4fe-403e-811a-dc5da0fa6661_f2ad8e1e-5162-4cf8-8dab-5cc5208cdb85.png" - }, - { - "name": "Grilled Chicken & Cheese Burger", - "price": "172.25", - "description": "A grilled chicken patty, topped with sliced cheese, spicy Habanero sauce, with some heat from jalapenos & crunch from onions. Contains: Sulphite, Soybeans, Egg, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/55a77d9e-cc28-4853-89c8-1ba3861f38c4_a378aafc-62b3-4328-a255-0f35f810966e.png" - }, - { - "name": "Corn & Cheese Burger", - "price": "166.32", - "description": "A juicy corn and cheese patty, topped with extra cheese, Cocktail sauce, with some heat from jalapenos & crunch from onions. Contains: Sulphite, Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/cb4d60c0-72c0-4694-8d41-3c745e253ea6_8262ec8c-e2ac-4c4f-8e52-36144a372851.png" - }, - { - "name": "Big Spicy Chicken Wrap", - "price": "240.57", - "description": "Tender and juicy chicken patty coated in spicy, crispy batter, topped with a creamy sauce, wrapped with lettuce, onions, tomatoes & cheese. A BIG indulgence.. Contains: Soybeans, Egg, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/22/e1fa4587-23ac-4613-af61-de659b066d19_14205d12-ea39-4894-aa85-59035020cecd.png" - }, - { - "name": "McCheese Burger Chicken", - "price": "282.15", - "description": "Double the indulgence with a sinfully oozing cheesy patty & flame-grilled chicken patty, along with chipotle sauce, shredded onion, jalapenos & lettuce.. Contains: Sulphite, Soybeans, Egg, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/b1184d5f-0785-4393-98a8-a712d280a045_027d0e63-e5d9-43f3-9c9c-0dc68dd1ece1.png" - }, - { - "name": "McCheese Burger Veg", - "price": "262.35", - "description": "Find pure indulgence in our Veg McCheese Burger, featuring a sinfully oozing cheesy veg patty, roasted chipotle sauce, jalapenos & lettuce.. Contains: Sulphite, Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/de84d4cc-6169-4235-942e-e4883a81c2e0_d90762c6-283f-46e5-a6ed-14ee3262bae0.png" - }, - { - "name": "McSpicy Premium Chicken Burger", - "price": "259.38", - "description": "A wholesome Spicy Chicken patty, Lettuce topped with Jalapenos and Cheese slice, Spicy Cocktail sauce & Cheese sauce. Contains: Sulphite, Soybeans, Egg, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/e1c10ab2-671b-4eac-aee8-88d9f96e005b_40169ccb-b849-4e0d-a54a-8938dd41ea34.png" - }, - { - "name": "McSpicy Premium Veg Burger", - "price": "249.47", - "description": "A wholesome Spicy Paneer patty, Lettuce topped with Jalapenos and Cheese slice, Spicy Cocktail sauce & Cheese sauce. Contains: Sulphite, Soybeans, Peanut, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/1a9faafd-b523-40a2-bdc6-f35191cfcf4a_0c462cb4-3843-4997-b813-dc34249b7c91.png" - }, - { - "name": "Chicken Maharaja Mac Burger", - "price": "268.28", - "description": "Savor our filling 11 layer burger! Double the indulgence with 2 juicy grilled chicken patties, along with jalapeños, onion, cheese, tomatoes, lettuce, and zesty Habanero sauce. . Contains: Sulphite, Soybeans, Egg, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/65a88cf4-7bcd-40f6-a09d-ec38c307c5d9_8993ea5b-f8e0-4d6c-9691-7f85adee2000.png" - }, - { - "name": "Chicken Surprise Burger", - "price": "75.23", - "description": "Introducing the new Chicken Surprise Burger which has the perfect balance of a crispy fried chicken patty, the crunch of onions and the richness of creamy sauce.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/0fbf18a1-5191-4cda-a09d-521a24c8c6ca_25cf57c6-48cc-47bd-b422-17e86b816422.png" - }, - { - "name": "McAloo Tikki Burger NONG", - "price": "69.30", - "description": "The World's favourite Indian burger with No Onion & No Garlic! Crispy aloo patty with delicious Tomato Mayo sauce!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/60311aec-07af-483d-b66a-ecae76edbd75_1407e287-7f07-4717-a6b1-14bff1a34961.png" - }, - { - "name": "Mexican McAloo Tikki Burger NONG", - "price": "74.24", - "description": "Your favourite McAloo Tikki with a fusion spin of Chipotle sauce. No Onion and No Garlic.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/c7d3076a-dfa7-4725-89cd-53754cecebee_1c11b7da-e00d-4e6b-bbe8-8ee847ee88a1.png" - }, - { - "name": "Crispy Veggie Burger", - "price": "198", - "description": "A flavorful patty made with a blend of 7 premium veggies, topped with zesty cocktail sauce, all served between soft, premium buns. Perfectly satisfying and full of flavor.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/7a3244bf-3091-4ae6-92e3-13be841a753e_b21f7a05-24b3-43f8-a592-beb23e6b69fa.png" - }, - { - "name": "Mc Crispy Chicken Burger", - "price": "221.76", - "description": "A crunchy, golden chicken thigh fillet, topped with fresh lettuce and creamy pepper mayo, all nestled between soft, toasted premium buns. Perfectly satisfying and full of flavor.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/df040551-263c-4074-86ee-68cb8cd393ba_3a5e53c6-601e-44a8-bf2e-590bffd7ee5e.png" - }, - { - "name": "McAloo Tikki Burger with Cheese", - "price": "98.05", - "description": "Savor the classic McAloo Tikki Burger, with an add-on cheese slice for a cheesy indulgence.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/10/30/0f0cbfd7-ee83-4e7c-bb6d-baab81e51646_4746c010-f82c-4e23-a43f-e36fe50b883b.png" - }, - { - "name": "Mexican McAloo Tikki with Cheese", - "price": "98.05", - "description": "Savor your favourite Mexican McAloo Tikki Burger, with an add-on cheese slice for a cheesy indulgence.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/10/30/d3cfbc30-4a7c-40a8-88a9-f714f3475523_14e7323b-24fe-4fbf-a517-68de64489103.png" - }, - { - "name": "Big Yummy Cheese Burger.", - "price": "349", - "description": "A spicy, cheesy indulgence, the Big Yummy Cheese Burger stacks a fiery paneer patty and a rich McCheese patty with crisp lettuce and smoky chipotle sauce on a Quarter Pound bun.. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/5c65bd4d-86ff-424c-800c-7d8b04aac86d_99e0d34c-cbba-41d9-bc31-a093237ba2af.png" - }, - { - "name": "Big Yummy Chicken Burger.", - "price": "349", - "description": "Crafted for true indulgence, tender grilled chicken patty meets the McCrispy chicken patty, elevated with crisp lettuce, jalapenos, and bold chipotle sauce.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/33234ab9-f10d-4605-89fc-349e0c7058bf_edba8f21-7c28-4fb6-88a2-dd3a86966175.png" - }, - { - "name": "Fries (Regular)", - "price": "88.11", - "description": "World Famous Fries, crispy, golden, lightly salted and fried to perfection! Also known as happiness.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/5a18fbbff67076c9a4457a6b220a55d9" - }, - { - "name": "Fries (Large)", - "price": "140.58", - "description": "World Famous Fries, crispy, golden, lightly salted and fried to perfection! Also known as happiness.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/a4b3002d0ea35bde5e5983f40e4ebfb4" - }, - { - "name": "Tomato Ketchup Sachet", - "price": "1", - "description": "Looking for a sauce to complement your meal? Look no further.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7db5533db29a4e9d2cc033f35c5572bc" - }, - { - "name": "9 Pc Chicken Nuggets", - "price": "221.74", - "description": "9 pieces of our iconic crispy, golden fried Chicken McNuggets!. Contains: Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/1ca7abb262e8880f5cb545d0d2f9bb9b" - }, - { - "name": "6 Pc Chicken Nuggets", - "price": "183.14", - "description": "6 pieces of our iconic crispy, golden fried Chicken McNuggets!. Contains: Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/44dc10c1099d7c366db9f5ce776878bd" - }, - { - "name": "Piri Piri Spice Mix", - "price": "23.80", - "description": "The perfect, taste bud tingling partner for our World Famous Fries. Shake Shake, and dive in!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/df3edfc74f610edff535324cc53a362a" - }, - { - "name": "Fries (Medium)", - "price": "120.78", - "description": "World Famous Fries, crispy, golden, lightly salted and fried to perfection! Also known as happiness.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/8a61e7fd97c454ea14d0750859fcebb8" - }, - { - "name": "Chilli Sauce Sachet", - "price": "2", - "description": "Looking for a sauce to complement your meal? Look no further.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f708dfc29c9624d8aef6e6ec30bde1c9" - }, - { - "name": "Veg Pizza McPuff", - "price": "64.35", - "description": "Crispy brown crust with a generous filling of rich tomato sauce, mixed with carrots, bell peppers, beans, onions and mozzarella. Served HOT.. Contains: Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/abe4b8cdf0f1bbfd1b9a7a05be3413e8" - }, - { - "name": "Classic Corn Cup", - "price": "90.08", - "description": "A delicious side of golden sweet kernels of corn in a cup.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9d67eae020425c4413acaf5af2a29dce" - }, - { - "name": "Fries (M) + Piri Piri Mix", - "price": "125", - "description": "Flat 15% Off on Fries (M) + Piri Piri Mix.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/15/b02b9b46-0d2b-46a6-aaa3-171c35101e11_24766c28-4fe9-4562-92d3-85c39d29c132.png" - }, - { - "name": "Cheesy Fries", - "price": "157.40", - "description": "The world famous, crispy golden Fries, served with delicious cheese sauce with a hint of spice. Contains cheese & mayonnaise. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/086cb28d-501d-42e5-a603-33e2d4493588_11348186-570f-44b8-b24e-88855455ba25.png" - }, - { - "name": "20 Pc Chicken Nuggets", - "price": "445.97", - "description": "20 pieces of our iconic crispy, golden fried Chicken McNuggets!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/1ca7abb262e8880f5cb545d0d2f9bb9b" - }, - { - "name": "Barbeque Sauce", - "price": "19.04", - "description": "Looking for a sauce to complement your meal? Look no further.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/ba0a188d45aecc3d4d187f340ea9df54" - }, - { - "name": "4 Pcs Chicken Nuggets", - "price": "109.88", - "description": "4 pieces of our iconic crispy, golden fried Chicken McNuggets!. Contains: Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/44dc10c1099d7c366db9f5ce776878bd" - }, - { - "name": "Mustard Sauce", - "price": "19.04", - "description": "Looking for a sauce to complement your meal? Look no further.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6c3aeffdbd544ea3ceae1e4b8ce3fc43" - }, - { - "name": "Spicy Sauce", - "price": "33.33", - "description": "Enjoy this spicy sauce that will add an extra kick to all your favourite items.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/dcc9ef6b-ceda-4b15-87af-ba2ad6c7de28_cfa5487a-811c-4c41-a770-af4ba6c5ebc8.png" - }, - { - "name": "Mango Smoothie", - "price": "210.86", - "description": "A delicious mix of mangoes, soft serve mix and blended ice. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/bb8470e1-5862-4844-bb47-ec0b2abdd752_845f1527-6bb0-46be-a1be-2fcc7532e813.png" - }, - { - "name": "Strawberry Green Tea (S)", - "price": "153.44", - "description": "Freshly-brewed refreshing tea with fruity Strawbery flavour.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/167dc0134bc1f4e8d7cb8e5c2a9dde5d" - }, - { - "name": "Moroccan Mint Green Tea (R )", - "price": "203.94", - "description": "Freshly-brewed refreshing tea with hint of Moroccon Mint flavour.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/2699c7c2130b4f50a09af3e294966b2e" - }, - { - "name": "Americano Coffee (R)", - "price": "190.07", - "description": "Refreshing cup of bold and robust espresso made with our signature 100% Arabica beans, combined with hot water.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/6a300499-efad-4a89-8fe4-ef6f6d3c1e2d_db248587-e3ea-4034-bbd7-33f9e483d6cb.png" - }, - { - "name": "Mixed Berry Smoothie", - "price": "210.86", - "description": "A mix of mixed berries, blended together with our creamy soft serve. Contains: Sulphite, Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/7a70ab81-ef1e-4e13-9c78-43ed2c62eaeb_3cd64c86-3a7b-41a6-9528-5fb5e2bbadc7.png" - }, - { - "name": "McCafe-Ice Coffee", - "price": "202.95", - "description": "Classic coffee poured over ice with soft servefor a refreshing pick-me-up. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/14/fd328038-532b-4df7-b6e9-b21bd6e8c70f_7e663943-ab8a-4f24-8dc0-d727dd503cd3.png" - }, - { - "name": "American Mud Pie Shake", - "price": "202.95", - "description": "Creamy and rich with chocolate and blended with nutty brownie bits for that extra thick goodness. Contains: Soybeans, Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/5ad3343d-a520-498d-a933-52104c624304_9f6cb499-0229-40dc-8b17-4c386f0cc287.png" - }, - { - "name": "Ice Americano Coffee", - "price": "180.18", - "description": "Signature Arabica espresso shot mixed with ice for an energizing experience.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7d89db9d67c537d666d838ddc1e0c44f" - }, - { - "name": "Americano Coffee (S)", - "price": "170.27", - "description": "Refreshing cup of bold and robust espresso made with our signature 100% Arabica beans, combined with hot water.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/6a300499-efad-4a89-8fe4-ef6f6d3c1e2d_db248587-e3ea-4034-bbd7-33f9e483d6cb.png" - }, - { - "name": "Coke", - "price": "101.97", - "description": "The perfect companion to your burger, fries and everything nice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/a1afed29afd8a2433b25cc47b83d01da" - }, - { - "name": "Mocha Coffee (R)", - "price": "236.60", - "description": "A delight of ground Arabica espresso, chocolate syrup and steamed milk. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/04a137420bf3febf861c4beed86d5702" - }, - { - "name": "Mocha Coffee (S)", - "price": "210.86", - "description": "A delight of ground Arabica espresso, chocolate syrup and steamed milk. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/71df07eb87d96824e2122f3412c8f743" - }, - { - "name": "Hot Chocolate (R)", - "price": "224.73", - "description": "Sinfully creamy chocolate whisked with silky streamed milk. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/1948679f-65df-4f93-a9cc-2ec796ca0818_6ad8b55d-e3cc-48ed-867d-511100b5735d.png" - }, - { - "name": "Latte Coffee (R)", - "price": "202.95", - "description": "A classic combination of the signature McCafe espresso, smooth milk, steamed and frothed. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/cee1ec0e10e25018572adcaf3a3c9e8c" - }, - { - "name": "Coke zero can", - "price": "66.66", - "description": "The perfect diet companion to your burger, fries and everything nice. Regular serving size, 300 Ml..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/8d6a37c4fc69bceb66b6a66690097190" - }, - { - "name": "Schweppes Water bottle", - "price": "66.66", - "description": "Quench your thirst with the Schweppes Water bottle.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/6/df44cf7c-aa13-46e2-9b38-e95a2f9faed4_d6c65089-cd93-473b-b711-aeac7fcf58b0.png" - }, - { - "name": "Fanta", - "price": "101.97", - "description": "Add a zest of refreshing orange to your meal..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7d662c96bc13c4ac33cea70c691f7f28" - }, - { - "name": "Mixed Fruit Beverage", - "price": "76.19", - "description": "Made with puree, pulp & juice from 6 delicious fruits. Contains: Soybeans, Peanut, Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/8e455d39bbbd8e4107b2099da51f3933" - }, - { - "name": "Sprite", - "price": "101.97", - "description": "The perfect companion to your burger, fries and everything nice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/46e03daf797857bfbce9f9fbb539a6aa" - }, - { - "name": "Cappuccino Coffee (R)", - "price": "199.98", - "description": "A refreshing espresso shot of 100% Arabica beans, topped with steamed milk froth. 473ml. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/4f45f0d8-111a-4d9c-a993-06d6858fdb06_cb5d79e9-c112-45e5-9e6a-e17d75acd8ac.png" - }, - { - "name": "Cappuccino Coffee (S)", - "price": "170.27", - "description": "A refreshing espresso shot of 100% Arabica beans, topped with steamed milk froth. 236ml. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/4f45f0d8-111a-4d9c-a993-06d6858fdb06_cb5d79e9-c112-45e5-9e6a-e17d75acd8ac.png" - }, - { - "name": "Berry Lemonade Regular", - "price": "141.57", - "description": "A refreshing drink, made with the delicious flavors of berries. 354 ml..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/14/4f0f852a-1e4a-4bd9-b0ec-69ccef3c6a34_0f4f0b64-1b1b-4a7d-98f9-e074fc96d1bd.png" - }, - { - "name": "Chocolate Flavoured Shake", - "price": "183.15", - "description": "The classic sinful Chocolate Flavoured Shake, a treat for anytime you need one. Now in new, convenient and delivery friendly packaging. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/463331b6-aa39-4d37-a8b2-c175aa20f723_14e888b6-2ebf-4d84-93a7-7b0576147bda.png" - }, - { - "name": "McCafe-Classic Coffee", - "price": "214.82", - "description": "An irrestible blend of our signature espresso and soft serve with whipped cream on top, a timeless combination! Now in a new, convenient and delivery friendly packaging.. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/5f9bdb36689a11cadb601a27b6fdef2d" - }, - { - "name": "Mocha Frappe", - "price": "278.19", - "description": "The perfect mix of indulgence and bold flavours. Enjoy a delicious blend of coffee, chocolate sauce, and soft serve. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/063e9cd747c621978ab4fddbb6d0a5ee" - }, - { - "name": "Cappuccino Small with Hazelnut", - "price": "183.15", - "description": "A delightful and aromatic coffee beverage that combines the robust flavor of espresso with the rich, nutty essence of hazelnut.. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/1793c61c-48f4-4785-b8a6-bc50eadb3b88_36ab0b13-0b17-459c-97a6-50dea3027b80.png" - }, - { - "name": "Ice Tea - Green Apple flavour", - "price": "182.16", - "description": "A perfect blend of aromatic teas, infused with green apple flavour .Now in a new, convenient and delivery friendly packaging.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/54243043b1b31fa715f38e6998a63e93" - }, - { - "name": "Strawberry Shake", - "price": "183.15", - "description": "An all time favourite treat bringing together the perfect blend of creamy vanilla soft serve and strawberry flavor.Now in new, convenient and delivery friendly packaging. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/b7d85483-4328-40a7-8cba-c408771d2482_99cab512-28fc-4b2c-b07f-87850002e79c.png" - }, - { - "name": "Cappuccino Small with French Vanilla", - "price": "182.16", - "description": "A popular coffee beverage that combines the smooth, creamy flavor of vanilla with the robust taste of espresso. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/e0a5c619-2df5-431f-9e21-670a358e8dbb_c2a2324f-ff95-44b3-adf9-dd6aee9696aa.png" - }, - { - "name": "Classic Coffee Regular with French Vanilla", - "price": "236", - "description": "a delightful and refreshing beverage that blends into the smooth, creamy essence of vanilla with the invigorating taste of chilled coffee.. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/5926eaba-bd6a-4af1-ba57-89bdb2fdfec6_c2507538-7597-4fc0-92e3-bdf57c81bda5.png" - }, - { - "name": "Classic Coffee Regular with Hazelnut", - "price": "233.63", - "description": "refreshing and delicious beverage that combines the rich, nutty taste of hazelnut with the cool, invigorating essence of cold coffee. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/b65446d8-bee6-484c-abdf-0326315c7e40_4c694dbc-e5e9-4998-b6ac-bd280fc6b5dc.png" - }, - { - "name": "Iced Coffee with French Vanilla", - "price": "223.74", - "description": "An ideal choice for those who enjoy a smooth, creamy vanilla twist to their iced coffee, providing a satisfying and refreshing pick-me-up. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/35867352-7802-4082-8b25-955878a6daa2_1d995db8-655f-45d3-9ad2-4469fe1301ae.png" - }, - { - "name": "Iced Coffee with Hazelnut", - "price": "223.74", - "description": "An ideal choice for those who enjoy a flavorful, nutty twist to their iced coffee, providing a satisfying and refreshing pick-me-up.. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/0619409e-e7cc-4b7c-a002-c05a2dff2ed8_f9bf368f-3dae-4af5-813d-4645384d24f7.png" - }, - { - "name": "Mint Lime Cooler", - "price": "101.97", - "description": "Refresh your senses with our invigorating Mint Lime Cooler. This revitalizing drink combines the sweetness of fresh lime juice and the subtle tang, perfectly balanced to quench your thirst and leave you feeling revitalized. Contains: Gluten, Milk, Peanut, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/a6b79a50-55c4-4b83-8b3c-d59c86ee1337_a3063ad4-fee2-4926-bf6d-ec98051b2249.png" - }, - { - "name": "Choco Crunch Cookie", - "price": "95", - "description": "Grab the choco crunch cookies packed with chocolate chips for the perfect crunch. Contains: Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/32ff88484a8607b6d740c1635b9ce09f" - }, - { - "name": "Hot Coffee Combo", - "price": "181", - "description": "In this combo choose any 1 hot coffee among- Cappuccino(s)/Latte(s)/Mocha(s)/Americano(s) and any 1 snacking item among- Choco crunch cookies/Oats Cookies/Choco Brownie/Blueberry muffin/ Signature croissant.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9aba6528e9c4f780dda18b5068349020" - }, - { - "name": "Indulge Choco Jar Dessert", - "price": "76", - "description": "Rich chocolate for pure indulgence to satify your sweet tooth..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/c7165efd872543e1648c21c930dafe5f" - }, - { - "name": "Cinnamon Raisin Cookie", - "price": "95", - "description": "Enjoy the wholesome flavours of this chewy and satisfying cookie. Contains: Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/fa5526832dfc6e236e6de7c322beae94" - }, - { - "name": "Cold Coffee Combo", - "price": "185", - "description": "In this combo choose any 1 coffee among- Cold Coffee ( R )/Iced Coffee ( R )/Iced Americano ( R ) and any 1 snacking item among- Choco crunch cookies/Oats Cookies/Choco Brownie/Blueberry muffin.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/cb24719daf96b58dd1be3bbf7b9fe372" - }, - { - "name": "Chocochip Muffin", - "price": "142", - "description": "Enjoy a dense chocochip muffin, with melty chocolate chips for a choco-lover's delight. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6c748c0b21f4a99593d1040021500430" - }, - { - "name": "Indulge Combo", - "price": "171", - "description": "Indulge in the perfect pairing of a classic cold coffee and a chocolate chip muffin, that balances the refreshing taste of chilled coffee with the sweet, comforting flavors of a freshly baked treat..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/ee2c7b9f-4426-4e70-9e98-bf8e8cc754ad_93eaaaa3-e498-4ca1-a77d-545a8006c8e2.png" - }, - { - "name": "Take a break Combo", - "price": "171", - "description": "Savor the harmonious pairing of a classic cappuccino with a cinnamon cookie, combining the bold, creamy flavors of coffee with the warm, spiced sweetness of a baked treat .", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/87a8807b-77e4-4e0d-a47e-af1cf2d2b96d_3230e601-af8f-4436-85cd-14c4e855240d.png" - }, - { - "name": "Treat Combo", - "price": "171", - "description": "Delight in the luxurious pairing of a chocolate jar dessert with a classic cappuccino, combining rich, creamy indulgence with the bold, aromatic flavors of espresso..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/9462ac0c-840e-44d1-a85d-bbea6f8268f1_a93dc44b-bf30-47b9-8cee-3a5589d2002e.png" - }, - { - "name": "Butter Croissant", - "price": "139", - "description": "Buttery, flaky croissant baked to golden perfection.Light, airy layers with a crisp outer shell.A classic French treat that melts in your mouth..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/da6cbf69-e422-4d02-9cc9-cb0452d04874_2153fe0c-389e-4842-b4c9-e0b78c450625.png" - }, - { - "name": "Butter Croissant + Cappuccino", - "price": "209", - "description": "Buttery croissant paired with a rich, frothy cappuccino.Warm, comforting, and perfectly balanced.A timeless duo for your anytime cravings..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/227a6aa3-f18c-4685-bb09-a76a01ec39a1_f347d737-b634-43d7-9b73-dc99bccde65f.png" - }, - { - "name": "Butter Croissant + Iced Coffee", - "price": "209", - "description": "Buttery, flaky croissant served with smooth, refreshing iced coffee. A classic combo that's light, crisp, and energizing. Perfect for a quick, satisfying bite..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/be3c8688-f975-4d1e-aad7-9229232bcc69_65c9a679-687d-4e66-9847-eb4d5c0f9825.png" - }, - { - "name": "Mcflurry Oreo ( S )", - "price": "104", - "description": "Delicious soft serve meets crumbled oreo cookies, a match made in dessert heaven. Perfect for one.. Contains: Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/a28369e386195be4071d9cf5078a438d" - }, - { - "name": "McFlurry Oreo ( M )", - "price": "129", - "description": "Delicious soft serve meets crumbled oreo cookies, a match made in dessert heaven. Share it, if you can.. Contains: Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f966500ed8b913a16cfdb25aab9244e4" - }, - { - "name": "Hot Fudge Sundae", - "price": "66", - "description": "A sinful delight, soft serve topped with delicious, gooey hot chocolate fudge. Always grab an extra spoon.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9c8958145495e8f2cf70470195f7834a" - }, - { - "name": "Strawberry Sundae", - "price": "66", - "description": "The cool vanilla soft serve ice cream with twirls of strawberry syrup.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/d7bd22aa47cffdcdde2d5b6223fde06e" - }, - { - "name": "Oreo Sundae ( M )", - "price": "72", - "description": "Enjoy the classic McFlurry Oreo goodness with a drizzle of hot fudge sauce with the Oreo Sundae!. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/3696da86802f534ba9ca68bd8be717ab" - }, - { - "name": "Black Forest McFlurry Medium", - "price": "139", - "description": "A sweet treat to suit your every mood. Contains: Soybeans, Peanut, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f513cc8c35cadd098835fb5b23c03561" - }, - { - "name": "Hot Fudge Brownie Sundae", - "price": "139", - "description": "Luscious chocolate brownie and hot-chocolate fudge to sweeten your day. Contains: Soybeans, Peanut, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6e00a57c6d8ceff6812a765c80e9ce74" - }, - { - "name": "Chocolate Overload McFlurry with Oreo Medium", - "price": "164.76", - "description": "Indulge in your chocolatey dreams with creamy soft serve, Oreo crumbs, a rich Hazelnut brownie, and two decadent chocolate sauces. Tempting, irresistible, and unforgettable. Contains: Gluten, Milk, Peanut, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/eb789769-9e60-4a84-9c64-90887ca79d7c_86154b6a-146c-47e9-9bbe-e685e4928e2e.png" - }, - { - "name": "Chocolate Overload McFlurry with Oreo Small", - "price": "134.28", - "description": "Indulge in your chocolatey dreams with creamy soft serve, Oreo crumbs, a rich Hazelnut brownie, and two decadent chocolate sauces. Tempting, irresistible, and unforgettable. Contains: Gluten, Milk, Peanut, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/431024a3-945d-4e6b-aafe-d35c410ac513_1b01e6dc-90cc-411d-a75e-9b98b42e178d.png" - } -] \ No newline at end of file diff --git a/firecrawl_menu.json b/firecrawl_menu.json deleted file mode 100644 index ce78f263..00000000 --- a/firecrawl_menu.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "items": [ - { - "name": "Cart", - "url": "https://www.swiggy.com/checkout" - }, - { - "name": "Sign In", - "url": null - }, - { - "name": "Help", - "url": "https://www.swiggy.com/support" - }, - { - "name": "OffersNEW", - "url": "https://www.swiggy.com/offers-near-me" - }, - { - "name": "Search", - "url": "https://www.swiggy.com/search" - }, - { - "name": "Swiggy Corporate", - "url": "https://www.swiggy.com/corporate" - } - ] -} \ No newline at end of file diff --git a/menu.json b/menu.json deleted file mode 100644 index 810be887..00000000 --- a/menu.json +++ /dev/null @@ -1,1688 +0,0 @@ -[ - { - "name": "Big Yummy Cheese Burger", - "price": "349", - "description": "A spicy, cheesy indulgence, the Big Yummy Cheese Burger stacks a fiery paneer patty and a rich McCheese patty with crisp lettuce and smoky chipotle sauce on a Quarter Pound bun.. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/916ad567-194a-449a-a9f3-c53acc0fa52e_4dfe7bfa-a200-4eab-9ffe-532236399652.png" - }, - { - "name": "Big Yummy Cheese Meal (M).", - "price": "424.76", - "description": "Double the indulgence, double the flavor: our Big Yummy Cheese Burger meal layers a spicy paneer patty and Cheese patty with crisp lettuce and smoky chipotle sauce, served with fries (M) and a beverage of your choice.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/7c1c4952-781a-4fe7-ad31-d6650fccbbdd_19f5f0fa-2cb3-405c-818a-457c84ba5a01.png" - }, - { - "name": "Big Yummy Chicken Burger", - "price": "349", - "description": "Crafted for true indulgence, tender grilled chicken patty meets the McCrispy chicken patty, elevated with crisp lettuce, jalapenos, and bold chipotle sauce.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/8bdf918d-2571-4f8c-a8f6-bcae8a257144_dbbba789-1e27-4bd0-a356-d7f5ed17c103.png" - }, - { - "name": "Big Yummy Chicken Meal (M).", - "price": "424.76", - "description": "Indulge in double the delight: our Big Yummy Chicken Burger meal pairs the tender grilled chicken patty and Crispy chicken patty with crisp lettuce, jalapeños, and bold chipotle sauce, served with fries (M) and a beverage of your choice ..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/be8e14ab-7861-4cf7-9550-d4a58f09d28c_958f0a00-5a1d-4041-825c-e196ae06d524.png" - }, - { - "name": "Cappuccino (S) + Iced Coffee (S)", - "price": "199.04", - "description": "Get the best coffee combo curated just for you!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/19/5aeea709-728c-43a2-ab00-4e8c754cec74_494372be-b929-496e-8db8-34e128746eb9.png" - }, - { - "name": "Veg Pizza McPuff + McSpicy Chicken Burger", - "price": "260", - "description": "Tender and juicy chicken patty coated in spicy, crispy batter topped with a creamy sauce and crispy shredded lettuce will have you craving for more. Served with Veg Pizza McPuff.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/fb4b9d4775505e82d05d6734ef3e2491" - }, - { - "name": "2 Cappuccino", - "price": "233.33", - "description": "2 Cappuccino (S).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/34aaf6ee-06e2-4c60-9950-8ad569bc5898_2639ffd2-47dd-447b-bbbe-eb391315666f.png" - }, - { - "name": "McChicken Burger + McSpicy Chicken Burger", - "price": "315.23", - "description": "The ultimate chicken combo made just for you. Get the top selling McChicken with the McSpicy Chicken Burger..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/006d51070b0ab9c839a293b87412541c" - }, - { - "name": "2 Iced Coffee", - "price": "233.33", - "description": "Enjoy 2 Iced Coffee.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/e36a9ed2-bfc0-4d36-bfef-297e2c74f991_90e415d8-c4f0-45c1-ad81-db8ef52a5f96.png" - }, - { - "name": "McVeggie Burger + McAloo Tikki Burger", - "price": "210.47", - "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, makes our iconic McVeggie and combo with our top selling McAloo Tikki Burger..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/1f4d583548597d41086df0c723560da7" - }, - { - "name": "Strawberry Shake + Fries (M)", - "price": "196", - "description": "Can't decide what to eat? We've got you covered. Get this snacking combo with Medium Fries and Strawberry Shake..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/74603316fc90ea3cd2b193ab491fbf53" - }, - { - "name": "McChicken Burger + Fries (M)", - "price": "244.76", - "description": "Tender and juicy chicken patty cooked to perfection, with creamy mayonnaise and crunchy lettuce adding flavour to each bite. Served with Medium Fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/a321db80-a223-4a90-9087-054154d27189_9168f1ee-991b-4d8c-8e82-64b2ea249943.png" - }, - { - "name": "McVeggie Burger + Fries (M)", - "price": "215.23", - "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns. Served with Medium Fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/d14cc495747a172686ebe43e675bc941" - }, - { - "name": "McAloo Tikki Burger + Veg Pizza McPuff + Coke", - "price": "190.47", - "description": "The ultimate veg combo made just for you. Get the top selling McAloo Tikki served with Veg Pizza McPuff and Coke..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9f0269a2d28f4918a3b07f63a487f26d" - }, - { - "name": "McAloo Tikki + Fries (R)", - "price": "115.23", - "description": "Aloo Tikki+ Fries (R).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/1ffa9f16-d7ce-48d8-946a-aaac56548c88_10b13615-190f-4f06-91cb-ac7499046fb8.png" - }, - { - "name": "Mexican McAloo Tikki Burger + Fries (R)", - "price": "120", - "description": "A fusion of international taste combined with your favourite aloo tikki patty, layered with shredded onion, and delicious Chipotle sauce. Served with Regular Fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7274d82212a758e597550e8c246fb2f7" - }, - { - "name": "McChicken Burger + Veg Pizza McPuff", - "price": "184.76", - "description": "Tender and juicy chicken patty cooked to perfection, with creamy mayonnaise and crunchy lettuce adding flavour to each bite. Served with Veg Pizza McPuff..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9a66b8ef66d780b9f83a0fc7cd434ded" - }, - { - "name": "McVeggie Burger + Veg Pizza McPuff", - "price": "195.23", - "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns. Served with Veg Pizza McPuff..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/a34eb684-5878-4578-8617-b06e24e46fba_36b736e7-3a91-4618-bf0f-ce60d45e55d2.png" - }, - { - "name": "McVeggie Burger + Fries (R)", - "price": "184.76", - "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns. Served with Regular fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/5e29050f-38f9-4d42-9388-be895f2ba84b_591326c1-a753-44b9-b80c-462196c67fd0.png" - }, - { - "name": "Mexican McAloo Tikki Burger + Fries (L)", - "price": "180", - "description": "A fusion of international taste combined with your favourite aloo tikki patty, layered with shredded onion, and delicious Chipotle sauce. Served with Large Fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f4be6e877d2567a0b585d4b16e53871e" - }, - { - "name": "McChicken Burger + Fries (L)", - "price": "250.47", - "description": "Tender and juicy chicken patty cooked to perfection, with creamy mayonnaise and crunchy lettuce adding flavour to each bite. Served with Large Fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/08e794cb-6520-4907-890c-27449181c9fb_e30fa88b-c5aa-4b71-8c84-135dc433c954.png" - }, - { - "name": "McVeggie Burger + Fries (L)", - "price": "250.47", - "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns. Served with Large fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/b02d421c-fae4-4408-870d-3b51c4e4a94d_0a64d6d8-9eea-452f-917e-988d7db846e2.png" - }, - { - "name": "2 Fries (R)", - "price": "120", - "description": "World Famous Fries, crispy, golden, lightly salted and fried to perfection! Double your happiness with this fries combo.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/4a170da5ecae92e11410a8fbb44c8476" - }, - { - "name": "2 McVeggie Burger", - "price": "270.47", - "description": "A delectable patty filled with potatoes, peas, carrots and tasty Indian spices. Topped with crispy lettuce, mayonnaise, and packed into toasted sesame buns makes our iconic McVeggie..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/f1edf611-08aa-4b91-b99c-f135eb70df66_ce7bec39-1633-4222-89a2-40013b5d9281.png" - }, - { - "name": "Grilled Chicken & Cheese Burger + Coke", - "price": "239.99", - "description": "Flat 15% Off on Grilled Chicken & Cheese Burger + Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/5/29/18f3ce07-00b5-487b-8608-56440efff007_2b241e0a-fad5-4be4-a848-81c124c95b8b.png" - }, - { - "name": "McAloo Tikki Burger + Veg Pizza McPuff + Fries (R)", - "price": "208.57", - "description": "Flat 15% Off on McAloo Tikki + Veg Pizza McPuff + Fries (R).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/6d7aef29-bae6-403b-a245-38c3371a5363_5dc1ef9c-1f17-4409-946b-b2d78b8710d4.png" - }, - { - "name": "McVeggie Burger + Fries (M) + Piri Piri Mix", - "price": "240", - "description": "Flat 15% Off on McVeggie Burger + Fries (M) + Piri Piri Mix.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/3011bf9d-01fb-4ff3-a2ca-832844aa0dd0_8c231aef-ff56-4dd2-82e5-743a6240c37b.png" - }, - { - "name": "McVeggie Burger + Veg Pizza McPuff + Fries (L)", - "price": "290.47", - "description": "Flat 15% Off on McVeggie Burger + Veg Pizza McPuff + Fries (L).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/5/29/0e4d8ee8-ee6b-4e12-9386-658dbcdc6be4_6810365e-3735-4b21-860b-923a416403be.png" - }, - { - "name": "6 Pc Chicken Nuggets + Fries (M) + Piri Piri Spice Mix", - "price": "247.99", - "description": "The best Non veg sides combo curated for you! Get 6 pc Chicken McNuggets + Fries M. Top it up with Piri Piri mix..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/8cba0938b4401e5c8cb2ccf3741b93c4" - }, - { - "name": "McAloo Tikki Burger + Veg Pizza McPuff + Piri Piri Spice Mix", - "price": "128", - "description": "Get India's favourite burger - McAloo Tikki along with Veg Pizza McPuff and spice it up with a Piri Piri Mix.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/46bcb1e486cbe6dcdb0487b063af58a6" - }, - { - "name": "Grilled Chicken & Cheese Burger + Veg Pizza McPuff", - "price": "196", - "description": "A delicious Grilled Chicken & Cheese Burger + a crispy brown, delicious Pizza McPuff.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/80983b8d-5d94-45f7-84a8-57f2477933de_83c6e531-c828-4ecd-a68a-ed518677fb66.png" - }, - { - "name": "Corn & Cheese Burger + Veg Pizza McPuff", - "price": "184.76", - "description": "A delicious Corn & Cheese Burger + a crispy brown, delicious Pizza McPuff.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/6d92a078-e41e-4126-b70c-b57538675892_c28b61a2-2733-4b3c-a6a2-7943fa109e24.png" - }, - { - "name": "Corn & Cheese Burger + Fries (R)", - "price": "210.47", - "description": "A delicious Corn & Cheese Burger + a side of crispy, golden, world famous fries ??.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/c66d0139-6560-4be9-9239-762f9e80a31a_b4576f89-8186-461d-ba99-28a31a0507f9.png" - }, - { - "name": "Corn & Cheese Burger + Coke", - "price": "239.99", - "description": "Flat 15% Off on Corn & Cheese Burger + Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/5/31/043061fe-2b66-49a4-bc65-b26232584003_ec753203-f35a-487f-bc87-8dcccd31ea3f.png" - }, - { - "name": "Chocolate Flavoured Shake+ Fries (M)", - "price": "196", - "description": "Can't decide what to eat? We've got you covered. Get this snacking combo with Medium Fries and Chocolate Flavoured Shake..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/46781c13d587e5f951ac1bbb39e57154" - }, - { - "name": "2 McFlurry Oreo (S)", - "price": "158", - "description": "Delicious soft serve meets crumbled oreo cookies, a match made in dessert heaven. Make it double with this combo!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/65f3574f53112e9d263dfa924b1f8fed" - }, - { - "name": "2 Hot Fudge Sundae", - "price": "156", - "description": "A sinful delight, soft serve topped with delicious, gooey hot chocolate fudge. So good you won't be able to stop at one!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f1238db9da73d8ec7a999792f35865d9" - }, - { - "name": "McSpicy Chicken Burger + Fries (M) + Piri Piri Spice Mix", - "price": "295.23", - "description": "Tender and juicy chicken patty coated in spicy, crispy batter topped with a creamy sauce and crispy shredded lettuce will have you craving for more. Served with the spicy piri piri mix and medium fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/e10982204687e18ee6541684365039b8" - }, - { - "name": "McSpicy Paneer Burger + Fries (M) + Piri Piri Spice Mix", - "price": "295.23", - "description": "Rich and filling cottage cheese patty coated in spicy, crispy batter topped with a creamy sauce and crispy shredded lettuce will have you craving for more. Served with the spicy piri piri mix and medium fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/87ce9779f986dcb21ab1fcfe794938d1" - }, - { - "name": "McSpicy Paneer + Cheesy Fries", - "price": "295.23", - "description": "Rich and filling cottage cheese patty coated in spicy, crispy batter topped with a creamy sauce and crispy shredded lettuce will have you craving for more. Served with Cheesy Fries..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/fcc2fb1635f8e14c69b57126014f0bd5" - }, - { - "name": "Black Forest Mcflurry (M) BOGO", - "price": "139", - "description": "Get 2 Black Forest McFlurry for the price of one!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/15/0319a787-bf68-4cca-a81c-f57d9d993918_9b42d2bc-f1bd-47d4-a5a0-6eb9944ec5cd.png" - }, - { - "name": "New McSaver Chicken Surprise", - "price": "119", - "description": "Enjoy a delicious combo of the new Chicken Surprise Burger with a beverage, now in a delivery friendly reusable bottle.. Contains: Sulphite, Soybeans, Milk, Egg, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/b8c0d92e-86b0-4efa-81c1-66be9b3f846b_ddd80150-1aa4-465c-8453-7f7e7650c6c9.png" - }, - { - "name": "New McSaver Chicken Nuggets (4 Pc)", - "price": "119", - "description": "Enjoy New McSaver Chicken Nuggets (4 Pc).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7bf83367ed61708817caefbc79a3c9eb" - }, - { - "name": "New McSaver McAloo Tikki", - "price": "119", - "description": "Enjoy New McSaver McAloo Tikki.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/ab4c47366f0e51ac0071f705b0f2d93e" - }, - { - "name": "New McSaver Pizza McPuff", - "price": "119", - "description": "Enjoy New McSaver Pizza McPuff.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/45fa406e76418771de26c37e8863fbb3" - }, - { - "name": "Chicken Surprise Burger + McChicken Burger", - "price": "204.76", - "description": "Enjoy the newly launched Chicken Surprise Burger with the iconic McChicken Burger.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/4/f1d9557a-6f86-4eb2-abd5-569f2e865de2_9b71c67a-45f1-41c1-9e1c-b80a934768da.png" - }, - { - "name": "Chicken Surprise Burger + Fries (M)", - "price": "170.47", - "description": "Enjoy the newly launched Chicken Surprise Burger with the iconic Fries (M).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/4/ccceb88b-2015-47a3-857b-21a9641d87ed_ad91cfe4-3727-4f23-a34e-4d09c8330709.png" - }, - { - "name": "Crispy Veggie Burger + Cheesy Fries", - "price": "320", - "description": "Feel the crunch with our newly launched Crispy Veggie Burger with Cheesy Fries.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/d81e09c2-e3c9-4b1e-862f-5b760c429a00_fa2d522d-2987-49c7-b924-965ddd970c0e.png" - }, - { - "name": "Crispy Veggie Burger + McAloo Tikki", - "price": "255.23", - "description": "Feel the crunch with our newly launched Crispy Veggie Burger + McAloo Tikki.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/476ed2c4-89b2-41d2-9903-be339a6a07e5_ff0d9d12-ecbd-43b9-97c9-dd3e3c20a5cb.png" - }, - { - "name": "Crispy Veggie Burger + Piri Piri Fries (M)", - "price": "320", - "description": "Feel the crunch with our newly launched Crispy Veggie Burger with Piri Piri Fries (M).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/abda9650-7216-4a8d-87ce-05342438db59_b501b87f-e104-466e-8af6-d3a9947e76ac.png" - }, - { - "name": "Mc Crispy Chicken Burger + Piri Piri Fries (M)", - "price": "360", - "description": "Feel the crunch with our newly launched McCrispy Chicken Burger with Piri Piri Fries (M).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/02bce46b-37fb-4aaa-a035-4daef4fe5350_1292d3ee-7a34-45a2-adc5-1e5b14b4752a.png" - }, - { - "name": "Mc Crispy Chicken Burger + Cheesy Fries", - "price": "370.47", - "description": "Feel the crunch with our newly launched McCrispy Chicken Burger with Cheesy Fries.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/170e8dd8-d9c1-4b67-ab84-cec5dd4a9e32_6d80a13b-024b-454f-b680-d62c7252cd4d.png" - }, - { - "name": "Chicken Surprise Burger + Cold Coffee", - "price": "266.66", - "description": "Start of your morning energetic and satisfied with our new exciting combo of - Chicken Surprise + Cold Coffee (R).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/10/30/01f2c1c4-e80b-48f0-9323-8e75775e08e3_3dd290a6-ed00-405b-a654-e46ed2854c81.png" - }, - { - "name": "McAloo Tikki Burger + Cold Coffee", - "price": "251.42", - "description": "Start of your morning energetic and satisfied with our new exciting combo of - McAloo Tikki +Cold Coffee (R).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/10/30/78de1f5b-7fa2-4e00-b81d-c6d6392cd9ea_6f6eaf41-6844-4349-b37c-49cb7be685b3.png" - }, - { - "name": "Choco Crunch Cookie + McAloo Tikki Burger", - "price": "144.76", - "description": "A crunchy, chocolatey delight meets the iconic Aloo Tikki Burger,sweet and savory, the perfect duo for your snack-time cravings!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/9229f5fb-a89c-49fc-ac40-45e59fdf69dd_e17fe978-1668-4829-812d-d00a1b46e971.png" - }, - { - "name": "Choco Crunch Cookie + McVeggie Burger", - "price": "223.80", - "description": "A crispy Choco Crunch Cookie and a hearty McVeggie Burger,your perfect balance of sweet indulgence and savory delight in every bite!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/61529326-5b1d-4917-b74f-1b3de9d4e8ef_8dbfb34e-5dd7-4a85-b0dd-3f38be677ff9.png" - }, - { - "name": "Lemon Ice Tea + Choco Crunch Cookie", - "price": "237.14", - "description": "A refreshing Lemon Iced Tea paired with a crunchy Choco Crunch Cookie, sweet, zesty, and perfectly balanced for a delightful treat!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/1b46fafc-e69f-4084-8d73-b47f4291e45b_4b0bdb80-9cd2-4b85-a6b5-13fc7348a3a3.png" - }, - { - "name": "Veg Pizza McPuff + Choco Crunch Cookie", - "price": "139.04", - "description": "A perfect snack duo, savoury, Veg Pizza McPuff paired with a crunchy, chocolatey Choco Crunch Cookie for a delicious treat!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/da76bf5a-f86c-44f7-9587-90ef5bdbe973_cf99d6eb-6cef-4f03-bd64-4e9b94523817.png" - }, - { - "name": "Butter Croissant + Cappuccino..", - "price": "209", - "description": "Buttery croissant paired with a rich, frothy cappuccino.Warm, comforting, and perfectly balanced.A timeless duo for your anytime cravings..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/227a6aa3-f18c-4685-bb09-a76a01ec39a1_f347d737-b634-43d7-9b73-dc99bccde65f.png" - }, - { - "name": "Butter Croissant + Iced Coffee.", - "price": "209", - "description": "Buttery, flaky croissant served with smooth, refreshing iced coffee. A classic combo that's light, crisp, and energizing. Perfect for a quick, satisfying bite..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/be3c8688-f975-4d1e-aad7-9229232bcc69_65c9a679-687d-4e66-9847-eb4d5c0f9825.png" - }, - { - "name": "1 Pc Crispy Fried Chicken", - "price": "108", - "description": "Enjoy the incredibly crunchy and juicy and Crispy Fried Chicken- 1 Pc.. Contains: Egg, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/4e7f77ef46d856205d3e4e4913ffc0e9" - }, - { - "name": "2 Crispy Fried Chicken + 2 McSpicy Fried Chicken + 2 Dips + 2 Coke", - "price": "548.99", - "description": "A combo of crunchy, juicy fried chicken and spicy, juicy McSpicy chicken, with 2 Dips and chilled Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/305180a8-d236-4f9f-9d52-852757dfc0f6_ab67db7f-eb5d-4519-88f5-a704b666db4e.png" - }, - { - "name": "2 Pc Crispy Fried Chicken", - "price": "219", - "description": "Enjoy 2 Pcs of the incredibly crunchy and juicy and Crispy Fried Chicken.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/07ed0b2a381f0a21d07888cf1b1216eb" - }, - { - "name": "1 McSpicy Fried Chk + 1 Crispy Fried Chk + 4 Wings + 2 Coke + 2 Dips", - "price": "532.37", - "description": "Juicy and spicy McSpicy chicken, crispy fried chicken, and wings with 2 Dips and Coke perfect for a flavorful meal..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/14c7ed0d-5164-4411-8c52-2151e93801be_4bf5a209-caed-4b44-91b8-48cd71455f2e.png" - }, - { - "name": "1 Pc McSpicy Fried Chicken", - "price": "114", - "description": "Try the new McSpicy Fried chicken that is juicy, crunchy and spicy to the last bite!. Contains: Sulphite, Soybeans, Peanut, Milk, Egg, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/ceb4e0a0-9bff-48f8-a7da-7e8f53c38b86_c9005198-cc85-420a-81ec-7e0bca0ca8ab.png" - }, - { - "name": "2 Pc McSpicy Chicken Wings", - "price": "93", - "description": "Enjoy the 2 pcs of the New McSpicy Chicken Wings. Spicy and crunchy, perfect for your chicken cravings. Contains: Sulphite, Soybeans, Peanut, Milk, Egg, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6e4918a1cf1113361edba3ed33519ffc" - }, - { - "name": "2 Pc McSpicy Fried Chicken", - "price": "217", - "description": "Try the new McSpicy Fried chicken- 2 pcs that is juicy, crunchy and spicy to the last bite!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/057b4bdb-9b88-4c9a-b54a-fc41d0ea1b5f_c6cd0222-69ae-4f30-b588-9741b82d9195.png" - }, - { - "name": "3 Pc McSpicy Fried Chicken Bucket + 1 Coke", - "price": "380.99", - "description": "Share your love for chicken with 3 pcs of McSpicy Fried Chicken with refreshing coke. The perfect meal for your catchup!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/9832febf-e35a-4575-89c1-dfdefd42bb92_42385d00-210d-4554-ac0f-236268e404a3.png" - }, - { - "name": "4 Pc McSpicy Chicken Wings", - "price": "185", - "description": "Enjoy the 4 pcs of the New McSpicy Chicken Wings. Spicy and crunchy, perfect for your chicken cravings.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7122300975cc9640e84cdc7e7c74e042" - }, - { - "name": "4 Pc McSpicy Fried Chicken Bucket", - "price": "455", - "description": "Share your love for chicken with 4 pcs of McSpicy Fried Chicken..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/5cacf92d-f847-4670-acc2-381f984790d1_60f07bc4-f161-4603-b473-527828f8df08.png" - }, - { - "name": "5 Pc McSpicy Fried Chicken Bucket", - "price": "590", - "description": "Share your love for chicken with 5 pcs of McSpicy Fried Chicken that is spicy to the last bite.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/11/16/1e617f1f-2573-475f-a50e-8c9dd02dd451_6ccb66a3-728b-4abb-8a03-4f3eed32da18.png" - }, - { - "name": "12 Pc Feast Chicken Bucket", - "price": "808.56", - "description": "Enjoy 12 pc bucket of 4 Pc McSpicy Fried Chicken + 4 Pc Crispy Fried Chicken+ 4 pc McSpicy Chicken Wings + 2 Medium Cokes + 2 Dips. (Serves 3-4).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/a09f66d7-3ad3-4f8c-9168-6a95849430f5_e360b6f1-659f-42c0-abc0-7f05b95496ce.png" - }, - { - "name": "Chicken Lover's Bucket", - "price": "599.04", - "description": "Enjoy this crunchy combination of 4 Pc McSpicy Chicken Wings + 2 Pc McSpicy Fried Chicken + 2 Pc Crispy Fried Chicken+ 2 Dips + 2 Cokes. A chicken lover's dream come true! (Serves 3-4).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/650f5998-10b0-4c07-bb3b-e68036242f11_c9b5e73c-4bbf-4c04-b918-dd7a32b7973b.png" - }, - { - "name": "4 Chicken Wings + 2 McSpicy Fried Chicken + 2 Coke + 2 Dips", - "price": "528.56", - "description": "Spicy, juicy McSpicy Chicken wings and 2 Pc McSpicy Fried chicken with 2 Dips, paired with 2 chilled cokes.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/42ff8854-3701-4b8c-aace-da723977c2e3_8b62f3cc-32eb-4e84-949e-375045125efc.png" - }, - { - "name": "4 McSpicy Fried Chicken Bucket + 2 Dips + 2 Coke", - "price": "532.37", - "description": "4 pieces of juicy, spicy McSpicy Fried Chicken with 2 Dips and the ultimate refreshment of chilled coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/305180a8-d236-4f9f-9d52-852757dfc0f6_ab67db7f-eb5d-4519-88f5-a704b666db4e.png" - }, - { - "name": "5 McSpicy Fried Chicken Bucket + 2 Dips + 2 Coke", - "price": "566.66", - "description": "5 pieces of juicy, spicy McSpicy Fried chicken with 2 Dips and 2 refreshing Cokes..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/29742264-8fee-4b8f-a8e2-52efb5d4edf9_314d6cbc-b4ea-4d92-92ea-5b7d1df70a9a.png" - }, - { - "name": "8 McSpicy Chicken Wings Bucket + 2 Coke + 1 Dip", - "price": "465.71", - "description": "Juicy, spicy McSpicy Chicken wings with 1 Dip and the ultimate refreshment of chilled coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/6/5/ea6a4090-5c1d-4b32-926b-5e1de33574ba_242accc3-d883-4434-9913-fb6c35574c9b.png" - }, - { - "name": "Chicken Surprise Burger with Multi-Millet Bun", - "price": "84.15", - "description": "Try the Chicken Surprise Burger in the new multi-millet bun! Enjoy the same tasty chicken patty you love, now sandwiched between a nutritious multi-millet bun.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/1fb900f6-71bb-4724-a507-830735f555c5_06c06358-bc36-469d-84e3-44e9504bb7d0.png" - }, - { - "name": "McAloo Tikki Burger with Multi-Millet Bun", - "price": "83.91", - "description": "Try your favourite McAloo Tikki Burger in a multi-millet bun! Enjoy the same tasty McAloo Tikki patty you love, now sandwiched between a nutritious millet bun..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/cfd7d779-b04d-43be-947e-a43df33ae119_a0de6b13-c9ac-489a-8416-e9b72ace4542.png" - }, - { - "name": "McChicken Burger with Multi-Millet Bun", - "price": "150.47", - "description": "Make a healthier choice with our McChicken Burger in a multi-millet bun! Same juicy chicken patty, now with a nutritious twist..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/bf0a9899-19a5-4fc9-9898-9e15098bac43_681fbed4-5354-4d5a-9d67-b59d274ea633.png" - }, - { - "name": "McSpicy Chicken Burger with Multi-Millet Bun", - "price": "221.76", - "description": "Feel the heat and feel good too! Try your McSpicy Chicken Burger in nutritious multi-millet bun..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/6eac1f61-cf7b-422a-bc6d-8c55bc1ff842_01e557a3-f5a2-4239-b06b-01ffd46ec154.png" - }, - { - "name": "McSpicy Paneer Burger with Multi-Millet Bun", - "price": "221.76", - "description": "Spice up your meal with a healthier bite! Try your McSpicy Paneer Burger with the nutritious multi-millet bun..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/70eb5956-3666-47f4-bcf8-1f5075207222_fd9e2a06-3f27-47e3-88aa-86895ffaa65e.png" - }, - { - "name": "McVeggie Burger with Multi-Millet Bun", - "price": "158.40", - "description": "Try your favorite McVeggie Burger in a nutritious multi-millet bun! A healthier twist on a classic favorite, with the same tasty veggie patty you love.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/397caee8-4da3-4143-98b1-e4dac155ab3e_0ed975df-7665-4f3a-ae16-8a7eb008afd6.png" - }, - { - "name": "Crispy Veggie Burger Protein Plus (1 Slice)", - "price": "226.71", - "description": "A flavourful patty made with a blend of 7 Premium veggies topped with zesty cocktail sauce now a protein slice to fuel you up, all served between soft premium buns. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/2fcff334-211c-4a2b-bee8-018ce9f10572_71ad5b2a-982d-407a-a833-8f3c1c6bf240.png" - }, - { - "name": "Crispy Veggie Burger Protein Plus (2 Slices)", - "price": "253.43", - "description": "A flavourful patty made with a blend of 7 Premium veggies topped with zesty cocktail sauce now a protein slice to fuel you up, all served between soft premium buns. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/e2c0ecf8-1ca0-447c-a56f-1b8758f7a406_314e4442-4db5-403e-b71c-ffa797622eee.png" - }, - { - "name": "Crispy Veggie Burger Protein Plus + Corn + Coke Zero", - "price": "399", - "description": "A flavourful patty made with a blend of 7 premium veggies, topped with zesty cocktail sauce and now a protein slice to fuel you up, all served between soft premium buns. Paired with corn and Coke Zero..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/d0ac58e8-e601-49e1-abf2-a5456a63f057_a8bb7584-906a-4ce1-891b-6e877f6b4543.png" - }, - { - "name": "McEgg Burger Protein Plus (1 Slice)", - "price": "89.10", - "description": "A steamed egg , spicy habanero sauce and onions and a tasty new protein slice. Simple, satisfying and powered with protein.. Contains: Gluten, Milk, Egg, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/0482392e-9b78-466d-93db-e653c77e0e35_55c40d44-9050-47eb-902e-5639c1423ee7.png" - }, - { - "name": "McEgg Burger Protein Plus (2 Slices)", - "price": "117.80", - "description": "A steamed egg , spicy habanero sauce and onions and a tasty new protein slice. Simple, satisfying and powered with protein.. Contains: Gluten, Milk, Egg, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/a5750dc8-8a1f-48a0-8fa2-83007ee4377c_c8501c16-90f4-4943-80f7-719f519fb918.png" - }, - { - "name": "McAloo Tikki Burger Protein Plus (1 Slice)", - "price": "89.10", - "description": "The OG Burger just got an upgrade with a tasty protein slice.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/6c4f7375-3424-4d01-8425-509381fb3015_25ebe6f6-2f23-4b3f-8dcb-02ea740f91e9.png" - }, - { - "name": "McAloo Tikki Burger Protein Plus (2 Slices)", - "price": "117.80", - "description": "The OG Burger just got an upgrade with a tasty protein slice.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/a95ad198-b6d8-43b0-9960-a1b8d8ceb6f2_12b5df65-ba9d-41ff-b762-88e391b121f8.png" - }, - { - "name": "McAloo Tikki Burger Protein Plus+ Corn + Coke Zero", - "price": "299", - "description": "The OG McAloo Tikki Burger just got an upgrade with a tasty protein slice. Served with buttery corn and a refreshing Coke Zero for a nostalgic yet balanced combo..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/da8beade-c083-4396-b4ff-289b647af81d_b0f6b946-7151-45e3-9fe0-e15a2c46c1d6.png" - }, - { - "name": "McCheese Chicken Burger Protein Plus (1 Slice)", - "price": "297", - "description": "Double the indulgence with sinfully oozing cheesey patty and flame grilled chicken patty , along with chipotle sauce , shredded onion , jalapenos , lettuce and now with a protein slice. Indulgent meets protein power.. Contains: Gluten, Egg, Milk, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/d32ea9af-d134-4ae9-8a1a-8dafddd53cb7_da16f662-ce98-4990-ab8c-566697743730.png" - }, - { - "name": "McCheese Chicken Burger Protein Plus (2 Slices)", - "price": "324.72", - "description": "Double the indulgence with sinfully oozing cheesey patty and flame grilled chicken patty , along ith chipotle sauce , shredded onion , jalapenos , lettuce and now with a protein slice. Indulgent meets protein power.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/e2b28b79-63df-43bb-8c41-240a076e0a3a_49dbc6df-3f9d-448c-bd94-377ebcc3a335.png" - }, - { - "name": "McCheese Chicken Burger Protein Plus + 4 Pc Chicken Nuggets+ Coke Zero", - "price": "449", - "description": "Double the indulgence with sinfully oozing cheesy patty and flame-grilled chicken patty, chipotle sauce, shredded onion, jalapenos, lettuce, and now a protein slice. Served with 4 Pc Chicken nuggets and Coke Zero. Indulgent meets protein power..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/f9aae1ec-6313-4684-aeb3-50de503fcf23_f5b200ed-60fa-42f6-928f-6b95327da3bb.png" - }, - { - "name": "McChicken Burger Protein Plus (1 Slice)", - "price": "185.13", - "description": "The classic McChicken you love, made more wholesome with a protein slice. Soft, savoury, and now protein rich.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/20fd3d59-0003-4e80-ad71-79ca8ae4d050_8bf1219b-a67d-4518-a115-167fa10bc440.png" - }, - { - "name": "McChicken Burger Protein Plus + 4 Pc Chicken Nuggets + Coke Zero", - "price": "349", - "description": "The classic McChicken you love, made more wholesome with a protein slice. Soft, savoury, and now protein-rich. Comes with 4 crispy Chicken nuggets and a chilled Coke Zero..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/afdcdd59-c5f0-44f1-9014-b28455ce0ecf_059ea761-4f24-4e04-b152-cdb82a04b366.png" - }, - { - "name": "McChicken Protein Burger Plus (2 Slices)", - "price": "212.84", - "description": "The classic McChicken you love, made more wholesome with a protein slice.Soft, savoury, and now protein-rich.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/1e1f281e-8378-4993-a7c1-d6f319a7bd17_768a7827-1fe8-43a6-ba35-a0ccb5a95ef5.png" - }, - { - "name": "McCrispy Chicken Burger Protein Plus (1 Slice)", - "price": "246.51", - "description": "A Crunchy , golden chicken thigh fillet , topped with fresh lettuce and creamy pepper mayo now also with a hearty protein slice all nestled between soft toasted premium buns.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/4f908fa6-2fa5-4367-9da9-9039cb5e72e0_c8570551-57ae-441a-81c9-64335392ac3b.png" - }, - { - "name": "McCrispy Chicken Burger Protein Plus (2 Slices)", - "price": "276.20", - "description": "A Crunchy , golden chicken thigh fillet , topped with fresh lettuce and creamy pepper mayo now also with a hearty protein slice all nestled between soft toasted premium buns.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/fc4e713d-5ee0-421e-9690-7c8b2c587e5a_2b710d54-3ae7-46fe-990c-140b34494dee.png" - }, - { - "name": "McCrispy Chicken Burger Protein Plus + 4 Pc Chicken Nugget + Coke Zero", - "price": "419", - "description": "A crunchy, golden chicken thigh fillet topped with fresh lettuce and creamy pepper mayo, now also with a hearty protein slice, all nestled between soft toasted premium buns. Comes with 4-piece nuggets and Coke Zero..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/fdb3d9b1-2262-4b18-9264-9bd00b3213ba_20858442-9b85-4728-b7b2-07d72349782e.png" - }, - { - "name": "McEgg Burger Protein Plus + 4 Pc Chicken Nuggets + Coke Zero", - "price": "299", - "description": "A steamed egg, spicy habanero sauce, onions, and a tasty new protein slice. Simple, satisfying, and powered with protein. Served with 4-piece chicken nuggets and Coke Zero..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/47092b74-fac3-48a2-9abc-1d43b975199f_d07e626a-6cb9-4664-8517-4ce7ba71b234.png" - }, - { - "name": "McSpicy Chicken Burger Protein Plus (1 Slice)", - "price": "225.72", - "description": "Indulge in our signature tender chicken patty coated in spicy crispy batter , topped with creamy sauce ,crispy lettuce and now with a new protein slice .. Contains: Gluten, Milk, Egg, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/79035335-855d-4b89-93e4-c063258aaf64_22e9ada8-6768-481d-83d3-7223bc119bce.png" - }, - { - "name": "McSpicy Chicken Burger Protein Plus (2 Slices)", - "price": "252.44", - "description": "Indulge in our signature tender chicken patty coated in spicy crispy batter , topped with creamy sauce ,crispy lettuce and now with a new protein slice .. Contains: Gluten, Egg, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/36e2e35c-86d5-4b0d-b573-325c7b4e5fd9_4bc54386-d43e-4d28-baca-d0da22bc3668.png" - }, - { - "name": "McSpicy Chicken Burger Protein Plus + 4 Pc Chicken Nuggets + Coke Zero", - "price": "399", - "description": "Indulge in our signature tender chicken patty coated in spicy crispy batter , topped with creamy sauce ,crispy lettuce and now with a new protein slice Served with 4-piece chicken nuggets and Coke Zero..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/a81bd599-05ad-4453-9ada-499d9cf45b29_1096d08c-d519-4133-babb-2f54a9b6e5f0.png" - }, - { - "name": "McSpicy Paneer Burger Protein Plus (1 Slice)", - "price": "226.71", - "description": "Indulge in rich and filling spicy paneer patty served with creamy sauce, crispy lettuce and now with a new protein slice. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/3ca8296e-9885-4df6-8621-5349298a150f_a2ad6b56-b6ae-4e4b-986c-abe75e052d75.png" - }, - { - "name": "McSpicy Paneer Burger Protein Plus (2 Slices)", - "price": "253.43", - "description": "Indulge in rich and filling spicy paneer patty served with creamy sauce, crispy lettuce and now with a new protein slice. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/c2561720-1064-4043-a143-3c9ae893e481_b09ff597-1b69-4098-ba04-6ab3f3b09c6f.png" - }, - { - "name": "McSpicy Paneer Burger Protein Plus + Corn + Coke Zero", - "price": "399", - "description": "Indulge in rich and filling spicy paneer patty served with creamy sauce, crispy lettuce and now with a new protein slice. Served with Corn and Coke Zero..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/d5c3f802-ad34-4086-ae66-731406a82c99_4b87555b-5e27-40da-b7f1-fd557212e344.png" - }, - { - "name": "McSpicy Premium Burger Veg Protein Plus (2 Slices)", - "price": "303.93", - "description": "A wholesome spicy paneer patty, lettuce topped with jalapenos and cheese slice and now with a protein-packed slice for that extra boost , spicy cocktail sauce and cheese sauce.. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/05949fc3-08a4-4e47-8fd1-16abc12a952c_0b80fed2-213d-4945-9ea7-38c818cdb6ef.png" - }, - { - "name": "McSpicy Premium Chicken Burger Protein Plus (1 Slice)", - "price": "287.10", - "description": "A wholesome spicy chicken patty lettuce topped with jalapenos and cheese slice plus an added protein slice , spicy cocktail sauce and cheese sauce.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/690ecde3-4611-4323-8436-a1dadfe2eb7b_fe9b5993-f422-4bae-8211-9715dd1d51d8.png" - }, - { - "name": "McSpicy Premium Chicken Burger Protein Plus (2 Slices)", - "price": "315.80", - "description": "A wholesome spicy chicken patty lettuce topped with jalapenos and cheese slice plus an added protein slice , spicy cocktail sauce and cheese sauce.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/80bf28f3-b317-4846-b678-95cfc857331a_d247479e-0bcb-4f5d-90da-b214d30e70a3.png" - }, - { - "name": "McSpicy Premium Chicken Protein Plus + 4 Pc Chicken Nugget + Coke Zero", - "price": "479", - "description": "A wholesome spicy chicken patty, lettuce topped with jalapenos and cheese slice, plus an added protein slice. Comes with spicy cocktail sauce and cheese sauce,served with 4 Pc crispy chicken nuggets and a Coke Zero..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/a512848a-d34d-44cf-a0db-f7e84781c63d_6d8fa5df-a1c6-42e6-b88b-929c59c4f50c.png" - }, - { - "name": "McSpicy Premium Veg Burger Protein Plus (1 Slice)", - "price": "276.20", - "description": "A wholesome spicy paneer patty, lettuce topped with jalapenos and cheese slice and now with a protein packed slice for that extra boost , spicy cocktail sauce and cheese sauce.. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/8768ff6a-ea10-4260-91e6-018695820d79_2399dc04-b793-4336-b2e4-a6240cdc5f78.png" - }, - { - "name": "McSpicy Premium Veg Burger Protein Plus + Corn + Coke Zero", - "price": "449", - "description": "A wholesome spicy paneer patty, lettuce topped with jalapenos and cheese slice, now with a protein-packed slice for that extra boost. Comes with spicy cocktail sauce, cheese sauce, buttery corn, and a chilled Coke Zero..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/d183fbeb-62dc-4b3d-936f-99ddc9dc5466_ce66dc12-da9f-479d-bd4a-c77fdb3dae2c.png" - }, - { - "name": "McVeggie Burger Protein Plus (1 Slice)", - "price": "185.13", - "description": "The classic McVeggie you love, made more wholesome with a protein slice.Soft, savoury, and now protein-rich.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/99aedaa8-836b-4893-b7f9-24b3303b6c94_781b8449-5bc9-42e5-9747-00bcec474deb.png" - }, - { - "name": "McVeggie Burger Protein Plus (2 Slices)", - "price": "212.88", - "description": "The classic McVeggie you love, made more wholesome with a protein slice.Soft, savoury, and now protein rich.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/31/630f74dd-841a-4641-bf7f-c23cd9e4e0d5_0fd861eb-43f6-46c1-9bfe-f8043c2aae6a.png" - }, - { - "name": "McVeggie Burger Protein Plus + Corn + Coke Zero", - "price": "339", - "description": "The classic McVeggie you love, made more wholesome with a protein slice. Soft, savoury, and now protein rich. Served with sweet corn and Coke Zero for a balanced meal..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/24/f448d30d-9bbe-4659-bbb4-14b22d93b0ae_0a188512-5619-4d62-980f-7db8e45f6ebe.png" - }, - { - "name": "Big Group Party Combo 6 Veg", - "price": "760.95", - "description": "Enjoy a Big Group Party Combo of McAloo + McVeggie + McSpicy Paneer + Mexican McAloo + Corn and Cheese + Crispy Veggie burger.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/427a977d-1e14-4fe9-a7ac-09a71f578514_e3089568-134d-45fe-b97e-59d756892f15.png" - }, - { - "name": "Big Group Party Combo for 6 Non- Veg", - "price": "856.19", - "description": "Enjoy a Big Group Party Combo of Surprise Chicken + McChicken + McSpicy Chicken + Grilled Chicken + McSpicy Premium + Mc Crispy Chicken Burger .", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/e1a453be-6ae4-4e82-8957-c43756d2d72a_d7cac204-1cb6-4b6d-addf-c0324435cc58.png" - }, - { - "name": "Big Group Party Combo1 for 4 Non- Veg", - "price": "475.23", - "description": "Save on your favourite Big Group Party Combo - Surprise Chicken + McChicken + McSpicy Chicken + Grilled Chicken Burger.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/d955dfa6-3a2c-46d2-b59c-e9e59b0939c3_1417a17c-bb7e-4e24-8931-3950cac2f074.png" - }, - { - "name": "Big Group Party Combo1 for 4 Veg", - "price": "475.23", - "description": "Get the best value in your Combo for 4 Save big on your favourite Big Group Party Combo-McAloo + McVeggie + McSpicy Paneer + Corn and Cheese Burger.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/5c723cbf-e7e2-48cc-a8fe-6f4e0eeea73d_899a4709-e9fb-4d9f-a997-973027fc0e7d.png" - }, - { - "name": "Big Group Party Combo2 for 4 Non-Veg", - "price": "522.85", - "description": "Your favorite party combo of 2 McChicken + 2 McSpicy Chicken Burger.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/b648ba7b-e82d-4d37-ab47-736fdb12cd90_19ef381a-3cc6-4b50-9e69-911f3656987f.png" - }, - { - "name": "2 Crispy Veggie Burger + Fries (L) + 2 Coke", - "price": "635.23", - "description": "Feel the crunch with Burger Combos for 2: 2 Crispy Veggie Burger + Fries (L)+ 2 Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/ce6551d8-829a-4dde-9131-cbf7818a6f26_a5b69cee-6377-4295-bbee-49725693c45d.png" - }, - { - "name": "Big Group Party Combo2 for 4 Veg", - "price": "522.85", - "description": "Your favorite party combo of 2 McVeggie + 2 McSpicy Paneer Burger.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/9/11/245c7235-500d-4212-84bd-f60f9aaf27be_73b24a53-d304-4db0-9336-061760ca17a9.png" - }, - { - "name": "2 Crispy Veggie Burger + 2 Fries (M) + Veg Pizza McPuff", - "price": "604.76", - "description": "Feel the crunch with Burger Combos for 2: 2 Crispy Veggie Burger + 2 Fries (M)+ Veg Pizza McPuff.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/ca2023e7-41e0-4233-a92c-c2bc52ef8ee5_33a0ea24-50ee-4a9b-b917-bbc2e6b5f37b.png" - }, - { - "name": "Crispy Veggie Burger + McVeggie Burger + Fries (M)", - "price": "424.76", - "description": "Feel the crunch with Crispy Veggie Burger+ McVeggie + Fries (M).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/c6dd67c1-0af9-4333-a4b5-3501e0ce4579_2fee69c9-f349-46c5-bdd9-8965b9f76687.png" - }, - { - "name": "Burger Combo for 2: McAloo Tikki", - "price": "364.76", - "description": "Stay home, stay safe and share a combo- 2 McAloo Tikki Burgers + 2 Fries (L).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/ea7ba594c7d77cb752de9a730fbcb3bf" - }, - { - "name": "6 Pc Chicken Nuggets + McChicken Burger + Coke", - "price": "375.22", - "description": "Tender and juicy chicken patty cooked to perfection, with creamy mayonnaise and crunchy lettuce adding flavour to each bite. Served with 6 Pc Nuggets and Coke..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/934194567f9c231dc46dccf2d4e6d415" - }, - { - "name": "Burger Combo for 2: McSpicy Chicken + McChicken", - "price": "464.76", - "description": "Flat 15% Off on McSpicy Chicken Burger + McChicken Burger + Fries (M).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/10ada13e-5724-487f-8ab6-fd07005859ad_a57d565d-0dfe-4424-bc5b-77b16143ad63.png" - }, - { - "name": "Burger Combo for 2: Corn & Cheese + McVeggie", - "price": "404.76", - "description": "Flat 15% Off on Corn & Cheese Burger +McVeggie Burger+Fries (M).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/08e9bc73-6774-41cf-96bb-ca817c4e23d3_e00ec89e-86f8-4531-956a-646082dc294c.png" - }, - { - "name": "Burger Combo for 2: McSpicy Chicken Burger with Pizza McPuff", - "price": "535.23", - "description": "Save big on your favourite sharing combo- 2 McSpicy Chicken Burger + 2 Fries (M) + Veg Pizza McPuff.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/e62aea3ba1cd5585a76004f59cd991e5" - }, - { - "name": "Burger Combo for 2: McSpicy Paneer + McAloo Tikki with Pizza McPuff", - "price": "427.61", - "description": "Get the best value in your meal for 2. Save big on your favourite sharing meal - McSpicy Paneer Burger + 2 Fries (M) + McAloo Tikki Burger + Veg Pizza McPuff.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/0ea8a2fddbbc17bc6239a9104963a3e8" - }, - { - "name": "Burger Combo for 2: McChicken Burger", - "price": "464.75", - "description": "Save big on your favourite sharing combo - 2 McChicken Burger + Fries (L) + 2 Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/7/7/177036e5-4afe-4076-b9cd-d7031f60ffe8_97687ef4-e242-4625-9b3c-398c60b8ddf2.png" - }, - { - "name": "Burger Combo for 2: McSpicy Chicken Burger", - "price": "548.56", - "description": "Save big on your favourite sharing combo - 2 McSpicy Chicken Burger + Fries (L) + 2 Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/98afaf26d81b15bec74cc356fe60cc13" - }, - { - "name": "Burger Combo for 2: McVeggie Burger", - "price": "424.75", - "description": "Save big on your favourite sharing combo - 2 McVeggie Burger + Fries (L) + 2 Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/09b3cb6130cfae15d486223c313fb6c6" - }, - { - "name": "2 Chicken Maharaja Mac Burger + 2 Coke + Fries (L) + McFlurry Oreo (M)", - "price": "670.47", - "description": "Enjoy 2 of the tallest burgers innovated by us. Created with chunky juicy grilled chicken patty paired along with fresh ingredients like jalapeno, onion, slice of cheese, tomatoes & crunchy lettuce dressed with the classical Habanero sauce. Served with Coke, Large Fries and a medium McFlurry Oreo.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/65c9c9b82c4d1f77a05dc4d89c9ead1d" - }, - { - "name": "Burger Combo for 2: Corn & Cheese Burger", - "price": "464.75", - "description": "Save big on your favourite sharing combo - 2 Corn and Cheese Burger + Fries (L) + 2 Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/847b562672e71c2352d92b797c0b0a4e" - }, - { - "name": "Burger Combo for 2: Grilled Chicken & Cheese", - "price": "495.23", - "description": "Save big on your favourite sharing combo - 2 Grilled Chicken and Cheese Burger + Fries (L) + 2 Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/1166a8baa3066342affb829ef0c428dd" - }, - { - "name": "2 Mc Crispy Chicken Burger + Fries (L) + 2 Coke", - "price": "724.75", - "description": "Feel the crunch with our Burger Combos for 2 : 2 McCrispy Chicken Burger + Fries (L)+ 2 Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/b4fda3b0-82c9-4d97-ab9c-c804b7d7893a_a14992d5-49cb-4035-827d-a43e40752840.png" - }, - { - "name": "Mc Crispy Chicken Burger + McChicken Burger + Fries (M)", - "price": "430.47", - "description": "Feel the crunch with McCrispy Chicken Burger+ McChicken + Fries (M).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/eaeeaf30-7bb6-4bef-9b78-643533a4520c_e54bb308-0447-44c2-9aaf-f36ce25f7939.png" - }, - { - "name": "Mc Crispy Chicken Burger + McSpicy Chicken Wings - 2 pc + Coke (M)", - "price": "415.23", - "description": "Feel the crunch with McCrispy Chicken Burger+ McSpicy Chicken Wings - 2 pc + Coke (M).", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/f8669731-b3e5-434d-8b45-269b75555b9b_3dfd8e01-5472-40db-a266-bb055036531b.png" - }, - { - "name": "McChicken Double Patty Burger Combo", - "price": "352.99", - "description": "Your favorite McChicken Burger double pattu burger + Fries (M) + Drink of your choice in a new, delivery friendly, resuable bottle..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/e8a8f43ee29a3b97d2fac37e89648eac" - }, - { - "name": "McSpicy Chicken Double Patty Burger combo", - "price": "418.99", - "description": "Your favorite McSpicy Chicken double patty Burger + Fries (M) + Drink of your choice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/91ed96b67df6e630a6830fc2e857b5b1" - }, - { - "name": "McVeggie Burger Happy Meal", - "price": "298.42", - "description": "Enjoy a combo of McVeggie Burger + Sweet Corn + B Natural Mixed Fruit Beverage + Book.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/15e7fc6b-f645-4ed9-a391-1e1edc452f9f_47ad6460-6af4-43b4-8365-35bb0b3fc078.png" - }, - { - "name": "McChicken Burger Happy Meal", - "price": "321.42", - "description": "Enjoy a combo of McChicken Burger + Sweet Corn + B Natural Mixed Fruit Beverage + Book.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/363ca5bf-bb04-4847-bc4e-5330a27d3874_8b5e46ed-61ba-4e8f-b291-2955af07eba0.png" - }, - { - "name": "McAloo Tikki Burger Happy Meal", - "price": "205.42", - "description": "Enjoy a combo of McAloo Tikki Burger + Sweet Corn + B Natural Mixed Fruit Beverage + Book.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/a9f2acb6-3fd3-4691-beb0-61439337f907_6733a4ee-3eb5-46da-9ab7-1b6943f68be7.png" - }, - { - "name": "Big Spicy Paneer Wrap Combo", - "price": "360.99", - "description": "Your favorite Big Spicy Paneer Wrap + Fries (M) + Drink of your choice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/22/116148a2-7f56-44af-bdd0-6bbb46f48baa_355bbd20-86d4-49b8-b3a9-d290772a9676.png" - }, - { - "name": "9 Pc Chicken Nuggets Combo", - "price": "388.98", - "description": "Enjoy your favorite Chicken McNuggets + Fries (M) + Drink of your choice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/4c56f086e500afe6b2025f9c46846e12" - }, - { - "name": "Mexican McAloo Tikki Burger Combo", - "price": "223.99", - "description": "Enjoy a delicious combo of Mexican McAloo Tikki Burger + Fries (M) + Beverage of your choice in a new, delivery friendly, reusable bottle..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/0140c49c7274cdb6af08053af1e6cc20" - }, - { - "name": "McEgg Burger Combo", - "price": "230.99", - "description": "Enjoy a combo of McEgg + Fries (M) + Drink of your Choice . Order now to experience a customizable, delicious meal..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/4f474c833930fa31d08ad2feed3414d8" - }, - { - "name": "McChicken Burger Combo", - "price": "314.99", - "description": "Your favorite McChicken Burger + Fries (M) + Drink of your choice in a new, delivery friendly, resuable bottle..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/b943fe56-212d-4366-a085-4e24d3532b30_3776df33-e18d-46c9-a566-c697897f1d16.png" - }, - { - "name": "McSpicy Chicken Burger Combo", - "price": "363.99", - "description": "Your favorite McSpicy Chicken Burger + Fries (M) + Drink of your choice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/214189a0-fd53-4f83-9d18-c3e53f2c017a_70748bf8-e587-47d6-886c-9b7ac57e84b3.png" - }, - { - "name": "McSpicy Paneer Burger Combo", - "price": "344.99", - "description": "Enjoy your favourite McSpicy Paneer Burger + Fries (M) + Drink of your Choice . Order now to experience a customizable, delicious combo.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/70fc7aa0-7f3b-418e-8ccd-5947b5f1aacd_055bbe77-fbe7-4200-84d3-4349475eb298.png" - }, - { - "name": "Veg Maharaja Mac Burger Combo", - "price": "398.99", - "description": "Enjoy a double decker Veg Maharaja Mac+ Fries (M) + Drink of your Choice . Order now to experience a customizable, delicious meal..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/d064fb5e-fb2e-4e1d-9515-18f26489f5b1_f6f49dba-2ae7-4ebf-8777-548c5ad4d799.png" - }, - { - "name": "McVeggie Burger Combo", - "price": "308.99", - "description": "Enjoy a combo of McVeggie + Fries (M) + Drink of your Choice in a new, delivery friendly, resuable bottle..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/5c9942c5-bc9d-4637-a312-cf51fe1d7aa8_e7e13473-9424-4eb8-87c3-368cfd084a2f.png" - }, - { - "name": "Big Spicy Chicken Wrap Combo", - "price": "379.99", - "description": "Your favorite Big Spicy Chicken Wrap + Fries (M) + Drink of your choice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/22/f631f834-aec2-410d-ab9d-7636cd53d7a0_8801a3c7-ad83-4f7c-bcb0-76101fadde91.png" - }, - { - "name": "Chicken Maharaja Mac Burger Combo", - "price": "398.99", - "description": "Enjoy a double decker Chicken Maharaja Mac + Fries (M) + Drink of your Choice . Order now to experience a customizable, delicious meal..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/ead090f2-5f80-4159-a335-d32658bcfc7c_8bcc5cd9-b22a-4f5d-8cde-0a2372c985c8.png" - }, - { - "name": "Grilled Cheese and Chicken Burger Combo", - "price": "310.47", - "description": "Enjoy a combo of Grilled Chicken & Cheese Burger + Fries (M) + Coke . Order now to experience a customizable, delicious meal..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/f6813404-54a4-492b-a03a-cf32d00ee1ae_c98c4761-69eb-4503-bde3-dffaafd43b15.png" - }, - { - "name": "Corn & cheese Burger Combo", - "price": "310.47", - "description": "Enjoy a combo of Corn & Cheese Burger + Fries (M) + Coke . Order now to experience a customizable, delicious meal..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/c8839f22-e525-44bb-8a50-8ee7cffecd26_cf4c0bcf-c1b9-4c0e-9109-03eb86abf4dd.png" - }, - { - "name": "Birthday Party Package - McChicken", - "price": "2169.14", - "description": "5 McChicken Burger + 5 Sweet Corn + 5 B Natural Mixed Fruit Beverage + 5 Soft Serve (M) + Book.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/15/37e0bc09-3e46-41d6-8146-66d6962ae2ab_0c91e294-cad2-4177-a2a9-86d6523bb811.png" - }, - { - "name": "Birthday Party Package - McVeggie", - "price": "2169.14", - "description": "5 McVeggie Burger + 5 Sweet Corn + 5 B Natural Mixed Fruit Beverage + 5 Soft Serve (M) + Book.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/15/0be6390f-e023-47f8-8bf4-9bb16d4995ce_2eceb8c2-6e90-47d7-97d6-93960391e668.png" - }, - { - "name": "McEgg Burger Happy Meal", - "price": "231.42", - "description": "Enjoy a combo of McEgg Burger + Sweet Corn + B Natural Mixed Fruit Beverage + Book.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/ada14e04-680d-44ef-bb05-180d0cc26ebc_b7bda720-9fcd-4bf4-97eb-bfb5a98e0239.png" - }, - { - "name": "McCheese Burger Veg Combo", - "price": "388.99", - "description": "Enjoy a deliciously filling meal of McCheese Veg Burger + Fries (M) + Beverage of your Choice in a delivery friendly, reusable bottle..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/22b25dd0-80ed-426b-a323-82c6b947612d_df827568-54f9-4329-af60-1fff38d105e9.png" - }, - { - "name": "McSpicy Premium Burger Chicken Combo", - "price": "398.99", - "description": "A deliciously filling meal of McSpicy Premium Chicken Burger + Fries (M) + Drink of your choice.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/28550356-fea3-47ce-bfa8-11222c78958a_8975c012-a121-42c3-92e7-719644761d83.png" - }, - { - "name": "McSpicy Premium Burger Veg Combo", - "price": "384.99", - "description": "A deliciously filling meal of McSpicy Premium Veg Burger + Fries (M) + Drink of your choice.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/14ce2cab-913c-4916-b98a-df2e5ea838ff_6d6208ca-4b76-4b87-a3fe-72c2db1081d8.png" - }, - { - "name": "McCheese Burger Chicken Combo", - "price": "388.99", - "description": "Enjoy a deliciously filling meal of McCheese Chicken Burger + Fries (M) + Beverage of your Choice in a delivery friendly, reusable bottle..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/b32eb9b3-7873-4f00-b601-0c92dd5a71ef_ec318a73-0e41-4277-8606-dc5c33b04533.png" - }, - { - "name": "McAloo Tikki Burger Combo", - "price": "204.99", - "description": "Enjoy a delicious combo of McAloo Tikki Burger + Fries (M) + Beverage of your choice in a new, delivery friendly, reusable bottle..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/b03a3ad7212fca0da40e90eed372ced9" - }, - { - "name": "McCheese Burger Veg Combo with Corn", - "price": "415.99", - "description": "Enjoy a combo of McCheese Burger Veg, Classic corn, McFlurry Oreo (Small) with a beverage of your choice in a delivery friendly, resuable bottle..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/9068eeb6-c774-4354-8c18-bf2ddcc94d10_631c39d1-d78c-4275-a14f-60670ce5aee4.png" - }, - { - "name": "2 Pc Chicken Nuggets Happy Meal", - "price": "211.42", - "description": "Enjoy a combo of 2 Pc Chicken Nuggets + Sweet Corn+ B Natural Mixed Fruit Beverage + Book.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/ac8a1d83-59c4-438f-bccb-edd601dacbf5_872c3881-77a1-43f7-82ca-2a929886045d.png" - }, - { - "name": "4 Pc Chicken Nuggets Happy Meal", - "price": "259.42", - "description": "Enjoy a combo of 4 Pc Chicken Nuggets + Sweet Corn+ B Natural Mixed Fruit Beverage + Book.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/10/10/fab04bb4-4f67-459a-85ef-5e9ae7861c88_f6761c27-26b3-4456-8abb-8f6c54274b34.png" - }, - { - "name": "Chicken McNuggets 6 Pcs Combo", - "price": "350.99", - "description": "Enjoy your favorite Chicken McNuggets + Fries (M) + Drink of your choice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6e7f9411ed67fe8d8873734af1e8d4e9" - }, - { - "name": "Chicken Surprise Burger + 4 Pc Chicken McNuggets + Coke", - "price": "259.99", - "description": "Enjoy the newly launched Chicken Surprise Burger with 4 Pc Chicken McNuggets and Coke.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/4/6c0288a4-943a-4bb6-a810-b14877c0ea8f_f56187da-4ae4-4a88-b1a5-e48e28d78087.png" - }, - { - "name": "Chicken Surprise Burger Combo", - "price": "238.09", - "description": "Chicken Surprise Burger + Fries (M) + Drink of your choice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/cb9833dd-5983-4445-bb1b-8d0e70b6c930_8928ae23-ddae-4d9a-abc5-e887d7d7868e.png" - }, - { - "name": "Crispy Veggie Burger Meal (M)", - "price": "326.99", - "description": "A flavorful patty with 7 premium veggies, zesty cocktail sauce, and soft buns, paired with crispy fries (M) and a refreshing Coke (M). A perfectly satisfying and full-flavored meal!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/eb4650ba-e950-4953-b1a7-c03a691ac0d2_6ca20c30-4c62-462a-b46d-9d1f21786b49.png" - }, - { - "name": "Mc Crispy Chicken Burger Meal (M)", - "price": "366.99", - "description": "A crunchy, golden chicken thigh fillet with fresh lettuce and creamy pepper mayo between soft, toasted premium buns, served with crispy fries (M) and a refreshing Coke (M). A perfectly satisfying and full-flavored meal!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/eedf0540-f558-4945-b996-994b1cd58048_d9af8cc8-384e-48c3-ab59-2facadfe574a.png" - }, - { - "name": "Choco Crunch Cookie + McAloo Tikki Burger + Lemon Ice Tea", - "price": "284.76", - "description": "Indulge in the perfect combo,crispy Choco Crunch Cookie, classic Aloo Tikki Burger, and refreshing Lemon Iced Tea. A delicious treat for your cravings, delivered fresh to your doorstep!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/b8ed02df-fb8f-4406-a443-398731aa9ef3_dafd8af6-e740-4987-8f4a-7bdc51dda4d8.png" - }, - { - "name": "Veg Pizza McPuff + Choco Crunch Cookie + Americano", - "price": "284.76", - "description": "A delightful trio, savoury Veg Pizza McPuff, crunchy Choco Crunch Cookie, and bold Americano, perfect for a satisfying snack break!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/25/9d81dad2-06a4-4402-9b07-2ed418963e16_f256697b-a444-4dfa-a9ff-d6448bb67b4b.png" - }, - { - "name": "Big Yummy Cheese Meal (M)", - "price": "424.76", - "description": "Double the indulgence, double the flavor: our Big Yummy Cheese Burger meal layers a spicy paneer patty and Cheese patty with crisp lettuce and smoky chipotle sauce, served with fries (M) and a beverage of your choice.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/cb892b9e-48b5-4127-b84f-b5e395961ddf_755bf2c0-e30a-4185-81e7-c35cbd07f483.png" - }, - { - "name": "Big Yummy Chicken Meal (M)", - "price": "424.76", - "description": "Indulge in double the delight: our Big Yummy Chicken Burger meal pairs the tender grilled chicken patty and Crispy chicken patty with crisp lettuce, jalapeños, and bold chipotle sauce, served with fries (M) and a beverage of your choice ..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/93313e0c-da3b-4490-839e-fd86b84c2644_b0076aa7-ef47-4f13-b72a-40f64744db95.png" - }, - { - "name": "McSpicy Chicken Double Patty Burger", - "price": "278.19", - "description": "Indulge in our signature tender double chicken patty, coated in spicy, crispy batter, topped with creamy sauce, and crispy lettuce.. Contains: Soybeans, Milk, Egg, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/314b5b5786f73746de4880602723a913" - }, - { - "name": "McChicken Double Patty Burger", - "price": "173.24", - "description": "Enjoy the classic, tender double chicken patty with creamy mayonnaise and lettuce in every bite. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/af88f46a82ef5e6a0feece86c349bb00" - }, - { - "name": "McVeggie Double Patty Burger", - "price": "186.12", - "description": "Savour your favorite spiced double veggie patty, lettuce, mayo, between toasted sesame buns in every bite. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/2d5062832f4d36c90e7dfe61ef48e85a" - }, - { - "name": "Mexican McAloo Tikki Double Patty Burger", - "price": "93.05", - "description": "A fusion of International taste combined with your favourite aloo tikki now with two patties. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/cda0e2d51420a95fad28ad728914b6de" - }, - { - "name": "McAloo Tikki Double Patty Burger", - "price": "88.11", - "description": "The World's favourite Indian burger! A crispy double Aloo patty, tomato mayo sauce & onions. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/ef569f74786e6344883a1decdd193229" - }, - { - "name": "McAloo Tikki Burger", - "price": "69.30", - "description": "The World's favourite Indian burger! A crispy Aloo patty, tomato mayo sauce & onions. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/b13811eeee71e578bc6ca89eca0ec87f" - }, - { - "name": "Big Spicy Paneer Wrap", - "price": "239.58", - "description": "Rich & filling cottage cheese patty coated in spicy crispy batter, topped with tom mayo sauce wrapped with lettuce, onions, tomatoes & cheese.. Contains: Sulphite, Soybeans, Peanut, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/22/198c3d14-3ce8-4105-8280-21577c26e944_779c4353-2f85-4515-94d2-208e90b830eb.png" - }, - { - "name": "McSpicy Chicken Burger", - "price": "226.71", - "description": "Indulge in our signature tender chicken patty, coated in spicy, crispy batter, topped with creamy sauce, and crispy lettuce.. Contains: Soybeans, Egg, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/dcdb436c-7b9f-4667-9b73-b8fa3215d7e2_9730340a-661b-49f4-a7d9-a8a89ffe988f.png" - }, - { - "name": "McSpicy Paneer Burger", - "price": "225.72", - "description": "Indulge in rich & filling spicy paneer patty served with creamy sauce, and crispy lettuce—irresistibly satisfying!. Contains: Sulphite, Soybeans, Peanut, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/fb912d21-ad9d-4332-b7cf-8f65d69e2c47_1fa5998c-b486-449a-9a88-2e61cf92ff77.png" - }, - { - "name": "Mexican McAloo Tikki Burger", - "price": "75.42", - "description": "Your favourite McAloo Tikki with a fusion spin with a Chipotle sauce & onions. Contains: Sulphite, Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/167aeccf27bab14940fa646c8328b1b4" - }, - { - "name": "McVeggie Burger", - "price": "153.44", - "description": "Savour your favorite spiced veggie patty, lettuce, mayo, between toasted sesame buns in every bite. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/2cf63c01-fef1-49b6-af70-d028bc79be7b_bfe88b73-33a9-489c-97f3-fb24631de1fc.png" - }, - { - "name": "McEgg Burger", - "price": "69.30", - "description": "A steamed egg, spicy Habanero sauce, & onions on toasted buns, a protein packed delight!. Contains: Soybeans, Milk, Egg, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/265c57f68b1a52f1cc4b63acf082d611" - }, - { - "name": "Veg Maharaja Mac Burger", - "price": "246.51", - "description": "Savor our filling 11 layer burger! Double the indulgence with 2 corn & cheese patties, along with jalapeños, onion, cheese, tomatoes, lettuce, and spicy Cocktail sauce. . Contains: Sulphite, Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/06354d09-be1b-406c-86b5-49dc9b5062d1_2f80f39e-c951-4ca6-8fca-a243a18c3448.png" - }, - { - "name": "McChicken Burger", - "price": "150.47", - "description": "Enjoy the classic, tender chicken patty with creamy mayonnaise and lettuce in every bite. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/c093ba63-c4fe-403e-811a-dc5da0fa6661_f2ad8e1e-5162-4cf8-8dab-5cc5208cdb85.png" - }, - { - "name": "Grilled Chicken & Cheese Burger", - "price": "172.25", - "description": "A grilled chicken patty, topped with sliced cheese, spicy Habanero sauce, with some heat from jalapenos & crunch from onions. Contains: Sulphite, Soybeans, Egg, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/55a77d9e-cc28-4853-89c8-1ba3861f38c4_a378aafc-62b3-4328-a255-0f35f810966e.png" - }, - { - "name": "Corn & Cheese Burger", - "price": "166.32", - "description": "A juicy corn and cheese patty, topped with extra cheese, Cocktail sauce, with some heat from jalapenos & crunch from onions. Contains: Sulphite, Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/cb4d60c0-72c0-4694-8d41-3c745e253ea6_8262ec8c-e2ac-4c4f-8e52-36144a372851.png" - }, - { - "name": "Big Spicy Chicken Wrap", - "price": "240.57", - "description": "Tender and juicy chicken patty coated in spicy, crispy batter, topped with a creamy sauce, wrapped with lettuce, onions, tomatoes & cheese. A BIG indulgence.. Contains: Soybeans, Egg, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/22/e1fa4587-23ac-4613-af61-de659b066d19_14205d12-ea39-4894-aa85-59035020cecd.png" - }, - { - "name": "McCheese Burger Chicken", - "price": "282.15", - "description": "Double the indulgence with a sinfully oozing cheesy patty & flame-grilled chicken patty, along with chipotle sauce, shredded onion, jalapenos & lettuce.. Contains: Sulphite, Soybeans, Egg, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/b1184d5f-0785-4393-98a8-a712d280a045_027d0e63-e5d9-43f3-9c9c-0dc68dd1ece1.png" - }, - { - "name": "McCheese Burger Veg", - "price": "262.35", - "description": "Find pure indulgence in our Veg McCheese Burger, featuring a sinfully oozing cheesy veg patty, roasted chipotle sauce, jalapenos & lettuce.. Contains: Sulphite, Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/de84d4cc-6169-4235-942e-e4883a81c2e0_d90762c6-283f-46e5-a6ed-14ee3262bae0.png" - }, - { - "name": "McSpicy Premium Chicken Burger", - "price": "259.38", - "description": "A wholesome Spicy Chicken patty, Lettuce topped with Jalapenos and Cheese slice, Spicy Cocktail sauce & Cheese sauce. Contains: Sulphite, Soybeans, Egg, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/e1c10ab2-671b-4eac-aee8-88d9f96e005b_40169ccb-b849-4e0d-a54a-8938dd41ea34.png" - }, - { - "name": "McSpicy Premium Veg Burger", - "price": "249.47", - "description": "A wholesome Spicy Paneer patty, Lettuce topped with Jalapenos and Cheese slice, Spicy Cocktail sauce & Cheese sauce. Contains: Sulphite, Soybeans, Peanut, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/29/1a9faafd-b523-40a2-bdc6-f35191cfcf4a_0c462cb4-3843-4997-b813-dc34249b7c91.png" - }, - { - "name": "Chicken Maharaja Mac Burger", - "price": "268.28", - "description": "Savor our filling 11 layer burger! Double the indulgence with 2 juicy grilled chicken patties, along with jalapeños, onion, cheese, tomatoes, lettuce, and zesty Habanero sauce. . Contains: Sulphite, Soybeans, Egg, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/21/65a88cf4-7bcd-40f6-a09d-ec38c307c5d9_8993ea5b-f8e0-4d6c-9691-7f85adee2000.png" - }, - { - "name": "Chicken Surprise Burger", - "price": "75.23", - "description": "Introducing the new Chicken Surprise Burger which has the perfect balance of a crispy fried chicken patty, the crunch of onions and the richness of creamy sauce.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/6/22/0fbf18a1-5191-4cda-a09d-521a24c8c6ca_25cf57c6-48cc-47bd-b422-17e86b816422.png" - }, - { - "name": "McAloo Tikki Burger NONG", - "price": "69.30", - "description": "The World's favourite Indian burger with No Onion & No Garlic! Crispy aloo patty with delicious Tomato Mayo sauce!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/60311aec-07af-483d-b66a-ecae76edbd75_1407e287-7f07-4717-a6b1-14bff1a34961.png" - }, - { - "name": "Mexican McAloo Tikki Burger NONG", - "price": "74.24", - "description": "Your favourite McAloo Tikki with a fusion spin of Chipotle sauce. No Onion and No Garlic.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/c7d3076a-dfa7-4725-89cd-53754cecebee_1c11b7da-e00d-4e6b-bbe8-8ee847ee88a1.png" - }, - { - "name": "Crispy Veggie Burger", - "price": "198", - "description": "A flavorful patty made with a blend of 7 premium veggies, topped with zesty cocktail sauce, all served between soft, premium buns. Perfectly satisfying and full of flavor.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/7a3244bf-3091-4ae6-92e3-13be841a753e_b21f7a05-24b3-43f8-a592-beb23e6b69fa.png" - }, - { - "name": "Mc Crispy Chicken Burger", - "price": "221.76", - "description": "A crunchy, golden chicken thigh fillet, topped with fresh lettuce and creamy pepper mayo, all nestled between soft, toasted premium buns. Perfectly satisfying and full of flavor.. Contains: Gluten, Milk, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/12/12/df040551-263c-4074-86ee-68cb8cd393ba_3a5e53c6-601e-44a8-bf2e-590bffd7ee5e.png" - }, - { - "name": "McAloo Tikki Burger with Cheese", - "price": "98.05", - "description": "Savor the classic McAloo Tikki Burger, with an add-on cheese slice for a cheesy indulgence.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/10/30/0f0cbfd7-ee83-4e7c-bb6d-baab81e51646_4746c010-f82c-4e23-a43f-e36fe50b883b.png" - }, - { - "name": "Mexican McAloo Tikki with Cheese", - "price": "98.05", - "description": "Savor your favourite Mexican McAloo Tikki Burger, with an add-on cheese slice for a cheesy indulgence.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/10/30/d3cfbc30-4a7c-40a8-88a9-f714f3475523_14e7323b-24fe-4fbf-a517-68de64489103.png" - }, - { - "name": "Big Yummy Cheese Burger.", - "price": "349", - "description": "A spicy, cheesy indulgence, the Big Yummy Cheese Burger stacks a fiery paneer patty and a rich McCheese patty with crisp lettuce and smoky chipotle sauce on a Quarter Pound bun.. Contains: Gluten, Milk, Peanut, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/5c65bd4d-86ff-424c-800c-7d8b04aac86d_99e0d34c-cbba-41d9-bc31-a093237ba2af.png" - }, - { - "name": "Big Yummy Chicken Burger.", - "price": "349", - "description": "Crafted for true indulgence, tender grilled chicken patty meets the McCrispy chicken patty, elevated with crisp lettuce, jalapenos, and bold chipotle sauce.. Contains: Gluten, Milk, Egg, Soybeans, Sulphite", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/33234ab9-f10d-4605-89fc-349e0c7058bf_edba8f21-7c28-4fb6-88a2-dd3a86966175.png" - }, - { - "name": "Fries (Regular)", - "price": "88.11", - "description": "World Famous Fries, crispy, golden, lightly salted and fried to perfection! Also known as happiness.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/5a18fbbff67076c9a4457a6b220a55d9" - }, - { - "name": "Fries (Large)", - "price": "140.58", - "description": "World Famous Fries, crispy, golden, lightly salted and fried to perfection! Also known as happiness.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/a4b3002d0ea35bde5e5983f40e4ebfb4" - }, - { - "name": "Tomato Ketchup Sachet", - "price": "1", - "description": "Looking for a sauce to complement your meal? Look no further.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7db5533db29a4e9d2cc033f35c5572bc" - }, - { - "name": "9 Pc Chicken Nuggets", - "price": "221.74", - "description": "9 pieces of our iconic crispy, golden fried Chicken McNuggets!. Contains: Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/1ca7abb262e8880f5cb545d0d2f9bb9b" - }, - { - "name": "6 Pc Chicken Nuggets", - "price": "183.14", - "description": "6 pieces of our iconic crispy, golden fried Chicken McNuggets!. Contains: Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/44dc10c1099d7c366db9f5ce776878bd" - }, - { - "name": "Piri Piri Spice Mix", - "price": "23.80", - "description": "The perfect, taste bud tingling partner for our World Famous Fries. Shake Shake, and dive in!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/df3edfc74f610edff535324cc53a362a" - }, - { - "name": "Fries (Medium)", - "price": "120.78", - "description": "World Famous Fries, crispy, golden, lightly salted and fried to perfection! Also known as happiness.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/8a61e7fd97c454ea14d0750859fcebb8" - }, - { - "name": "Chilli Sauce Sachet", - "price": "2", - "description": "Looking for a sauce to complement your meal? Look no further.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f708dfc29c9624d8aef6e6ec30bde1c9" - }, - { - "name": "Veg Pizza McPuff", - "price": "64.35", - "description": "Crispy brown crust with a generous filling of rich tomato sauce, mixed with carrots, bell peppers, beans, onions and mozzarella. Served HOT.. Contains: Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/abe4b8cdf0f1bbfd1b9a7a05be3413e8" - }, - { - "name": "Classic Corn Cup", - "price": "90.08", - "description": "A delicious side of golden sweet kernels of corn in a cup.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9d67eae020425c4413acaf5af2a29dce" - }, - { - "name": "Fries (M) + Piri Piri Mix", - "price": "125", - "description": "Flat 15% Off on Fries (M) + Piri Piri Mix.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/15/b02b9b46-0d2b-46a6-aaa3-171c35101e11_24766c28-4fe9-4562-92d3-85c39d29c132.png" - }, - { - "name": "Cheesy Fries", - "price": "157.40", - "description": "The world famous, crispy golden Fries, served with delicious cheese sauce with a hint of spice. Contains cheese & mayonnaise. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/30/086cb28d-501d-42e5-a603-33e2d4493588_11348186-570f-44b8-b24e-88855455ba25.png" - }, - { - "name": "20 Pc Chicken Nuggets", - "price": "445.97", - "description": "20 pieces of our iconic crispy, golden fried Chicken McNuggets!.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/1ca7abb262e8880f5cb545d0d2f9bb9b" - }, - { - "name": "Barbeque Sauce", - "price": "19.04", - "description": "Looking for a sauce to complement your meal? Look no further.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/ba0a188d45aecc3d4d187f340ea9df54" - }, - { - "name": "4 Pcs Chicken Nuggets", - "price": "109.88", - "description": "4 pieces of our iconic crispy, golden fried Chicken McNuggets!. Contains: Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/44dc10c1099d7c366db9f5ce776878bd" - }, - { - "name": "Mustard Sauce", - "price": "19.04", - "description": "Looking for a sauce to complement your meal? Look no further.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6c3aeffdbd544ea3ceae1e4b8ce3fc43" - }, - { - "name": "Spicy Sauce", - "price": "33.33", - "description": "Enjoy this spicy sauce that will add an extra kick to all your favourite items.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/dcc9ef6b-ceda-4b15-87af-ba2ad6c7de28_cfa5487a-811c-4c41-a770-af4ba6c5ebc8.png" - }, - { - "name": "Mango Smoothie", - "price": "210.86", - "description": "A delicious mix of mangoes, soft serve mix and blended ice. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/bb8470e1-5862-4844-bb47-ec0b2abdd752_845f1527-6bb0-46be-a1be-2fcc7532e813.png" - }, - { - "name": "Strawberry Green Tea (S)", - "price": "153.44", - "description": "Freshly-brewed refreshing tea with fruity Strawbery flavour.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/167dc0134bc1f4e8d7cb8e5c2a9dde5d" - }, - { - "name": "Moroccan Mint Green Tea (R )", - "price": "203.94", - "description": "Freshly-brewed refreshing tea with hint of Moroccon Mint flavour.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/2699c7c2130b4f50a09af3e294966b2e" - }, - { - "name": "Americano Coffee (R)", - "price": "190.07", - "description": "Refreshing cup of bold and robust espresso made with our signature 100% Arabica beans, combined with hot water.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/6a300499-efad-4a89-8fe4-ef6f6d3c1e2d_db248587-e3ea-4034-bbd7-33f9e483d6cb.png" - }, - { - "name": "Mixed Berry Smoothie", - "price": "210.86", - "description": "A mix of mixed berries, blended together with our creamy soft serve. Contains: Sulphite, Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/7a70ab81-ef1e-4e13-9c78-43ed2c62eaeb_3cd64c86-3a7b-41a6-9528-5fb5e2bbadc7.png" - }, - { - "name": "McCafe-Ice Coffee", - "price": "202.95", - "description": "Classic coffee poured over ice with soft servefor a refreshing pick-me-up. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/14/fd328038-532b-4df7-b6e9-b21bd6e8c70f_7e663943-ab8a-4f24-8dc0-d727dd503cd3.png" - }, - { - "name": "American Mud Pie Shake", - "price": "202.95", - "description": "Creamy and rich with chocolate and blended with nutty brownie bits for that extra thick goodness. Contains: Soybeans, Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/5ad3343d-a520-498d-a933-52104c624304_9f6cb499-0229-40dc-8b17-4c386f0cc287.png" - }, - { - "name": "Ice Americano Coffee", - "price": "180.18", - "description": "Signature Arabica espresso shot mixed with ice for an energizing experience.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7d89db9d67c537d666d838ddc1e0c44f" - }, - { - "name": "Americano Coffee (S)", - "price": "170.27", - "description": "Refreshing cup of bold and robust espresso made with our signature 100% Arabica beans, combined with hot water.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/6a300499-efad-4a89-8fe4-ef6f6d3c1e2d_db248587-e3ea-4034-bbd7-33f9e483d6cb.png" - }, - { - "name": "Coke", - "price": "101.97", - "description": "The perfect companion to your burger, fries and everything nice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/a1afed29afd8a2433b25cc47b83d01da" - }, - { - "name": "Mocha Coffee (R)", - "price": "236.60", - "description": "A delight of ground Arabica espresso, chocolate syrup and steamed milk. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/04a137420bf3febf861c4beed86d5702" - }, - { - "name": "Mocha Coffee (S)", - "price": "210.86", - "description": "A delight of ground Arabica espresso, chocolate syrup and steamed milk. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/71df07eb87d96824e2122f3412c8f743" - }, - { - "name": "Hot Chocolate (R)", - "price": "224.73", - "description": "Sinfully creamy chocolate whisked with silky streamed milk. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/15/1948679f-65df-4f93-a9cc-2ec796ca0818_6ad8b55d-e3cc-48ed-867d-511100b5735d.png" - }, - { - "name": "Latte Coffee (R)", - "price": "202.95", - "description": "A classic combination of the signature McCafe espresso, smooth milk, steamed and frothed. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/cee1ec0e10e25018572adcaf3a3c9e8c" - }, - { - "name": "Coke zero can", - "price": "66.66", - "description": "The perfect diet companion to your burger, fries and everything nice. Regular serving size, 300 Ml..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/8d6a37c4fc69bceb66b6a66690097190" - }, - { - "name": "Schweppes Water bottle", - "price": "66.66", - "description": "Quench your thirst with the Schweppes Water bottle.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/2/6/df44cf7c-aa13-46e2-9b38-e95a2f9faed4_d6c65089-cd93-473b-b711-aeac7fcf58b0.png" - }, - { - "name": "Fanta", - "price": "101.97", - "description": "Add a zest of refreshing orange to your meal..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/7d662c96bc13c4ac33cea70c691f7f28" - }, - { - "name": "Mixed Fruit Beverage", - "price": "76.19", - "description": "Made with puree, pulp & juice from 6 delicious fruits. Contains: Soybeans, Peanut, Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/8e455d39bbbd8e4107b2099da51f3933" - }, - { - "name": "Sprite", - "price": "101.97", - "description": "The perfect companion to your burger, fries and everything nice..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/46e03daf797857bfbce9f9fbb539a6aa" - }, - { - "name": "Cappuccino Coffee (R)", - "price": "199.98", - "description": "A refreshing espresso shot of 100% Arabica beans, topped with steamed milk froth. 473ml. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/4f45f0d8-111a-4d9c-a993-06d6858fdb06_cb5d79e9-c112-45e5-9e6a-e17d75acd8ac.png" - }, - { - "name": "Cappuccino Coffee (S)", - "price": "170.27", - "description": "A refreshing espresso shot of 100% Arabica beans, topped with steamed milk froth. 236ml. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/4f45f0d8-111a-4d9c-a993-06d6858fdb06_cb5d79e9-c112-45e5-9e6a-e17d75acd8ac.png" - }, - { - "name": "Berry Lemonade Regular", - "price": "141.57", - "description": "A refreshing drink, made with the delicious flavors of berries. 354 ml..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/8/14/4f0f852a-1e4a-4bd9-b0ec-69ccef3c6a34_0f4f0b64-1b1b-4a7d-98f9-e074fc96d1bd.png" - }, - { - "name": "Chocolate Flavoured Shake", - "price": "183.15", - "description": "The classic sinful Chocolate Flavoured Shake, a treat for anytime you need one. Now in new, convenient and delivery friendly packaging. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/463331b6-aa39-4d37-a8b2-c175aa20f723_14e888b6-2ebf-4d84-93a7-7b0576147bda.png" - }, - { - "name": "McCafe-Classic Coffee", - "price": "214.82", - "description": "An irrestible blend of our signature espresso and soft serve with whipped cream on top, a timeless combination! Now in a new, convenient and delivery friendly packaging.. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/5f9bdb36689a11cadb601a27b6fdef2d" - }, - { - "name": "Mocha Frappe", - "price": "278.19", - "description": "The perfect mix of indulgence and bold flavours. Enjoy a delicious blend of coffee, chocolate sauce, and soft serve. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/063e9cd747c621978ab4fddbb6d0a5ee" - }, - { - "name": "Cappuccino Small with Hazelnut", - "price": "183.15", - "description": "A delightful and aromatic coffee beverage that combines the robust flavor of espresso with the rich, nutty essence of hazelnut.. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/1793c61c-48f4-4785-b8a6-bc50eadb3b88_36ab0b13-0b17-459c-97a6-50dea3027b80.png" - }, - { - "name": "Ice Tea - Green Apple flavour", - "price": "182.16", - "description": "A perfect blend of aromatic teas, infused with green apple flavour .Now in a new, convenient and delivery friendly packaging.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/54243043b1b31fa715f38e6998a63e93" - }, - { - "name": "Strawberry Shake", - "price": "183.15", - "description": "An all time favourite treat bringing together the perfect blend of creamy vanilla soft serve and strawberry flavor.Now in new, convenient and delivery friendly packaging. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/3/14/b7d85483-4328-40a7-8cba-c408771d2482_99cab512-28fc-4b2c-b07f-87850002e79c.png" - }, - { - "name": "Cappuccino Small with French Vanilla", - "price": "182.16", - "description": "A popular coffee beverage that combines the smooth, creamy flavor of vanilla with the robust taste of espresso. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/e0a5c619-2df5-431f-9e21-670a358e8dbb_c2a2324f-ff95-44b3-adf9-dd6aee9696aa.png" - }, - { - "name": "Classic Coffee Regular with French Vanilla", - "price": "236", - "description": "a delightful and refreshing beverage that blends into the smooth, creamy essence of vanilla with the invigorating taste of chilled coffee.. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/5926eaba-bd6a-4af1-ba57-89bdb2fdfec6_c2507538-7597-4fc0-92e3-bdf57c81bda5.png" - }, - { - "name": "Classic Coffee Regular with Hazelnut", - "price": "233.63", - "description": "refreshing and delicious beverage that combines the rich, nutty taste of hazelnut with the cool, invigorating essence of cold coffee. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/b65446d8-bee6-484c-abdf-0326315c7e40_4c694dbc-e5e9-4998-b6ac-bd280fc6b5dc.png" - }, - { - "name": "Iced Coffee with French Vanilla", - "price": "223.74", - "description": "An ideal choice for those who enjoy a smooth, creamy vanilla twist to their iced coffee, providing a satisfying and refreshing pick-me-up. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/35867352-7802-4082-8b25-955878a6daa2_1d995db8-655f-45d3-9ad2-4469fe1301ae.png" - }, - { - "name": "Iced Coffee with Hazelnut", - "price": "223.74", - "description": "An ideal choice for those who enjoy a flavorful, nutty twist to their iced coffee, providing a satisfying and refreshing pick-me-up.. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/0619409e-e7cc-4b7c-a002-c05a2dff2ed8_f9bf368f-3dae-4af5-813d-4645384d24f7.png" - }, - { - "name": "Mint Lime Cooler", - "price": "101.97", - "description": "Refresh your senses with our invigorating Mint Lime Cooler. This revitalizing drink combines the sweetness of fresh lime juice and the subtle tang, perfectly balanced to quench your thirst and leave you feeling revitalized. Contains: Gluten, Milk, Peanut, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/8/22/a6b79a50-55c4-4b83-8b3c-d59c86ee1337_a3063ad4-fee2-4926-bf6d-ec98051b2249.png" - }, - { - "name": "Choco Crunch Cookie", - "price": "95", - "description": "Grab the choco crunch cookies packed with chocolate chips for the perfect crunch. Contains: Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/32ff88484a8607b6d740c1635b9ce09f" - }, - { - "name": "Hot Coffee Combo", - "price": "181", - "description": "In this combo choose any 1 hot coffee among- Cappuccino(s)/Latte(s)/Mocha(s)/Americano(s) and any 1 snacking item among- Choco crunch cookies/Oats Cookies/Choco Brownie/Blueberry muffin/ Signature croissant.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9aba6528e9c4f780dda18b5068349020" - }, - { - "name": "Indulge Choco Jar Dessert", - "price": "76", - "description": "Rich chocolate for pure indulgence to satify your sweet tooth..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/c7165efd872543e1648c21c930dafe5f" - }, - { - "name": "Cinnamon Raisin Cookie", - "price": "95", - "description": "Enjoy the wholesome flavours of this chewy and satisfying cookie. Contains: Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/fa5526832dfc6e236e6de7c322beae94" - }, - { - "name": "Cold Coffee Combo", - "price": "185", - "description": "In this combo choose any 1 coffee among- Cold Coffee ( R )/Iced Coffee ( R )/Iced Americano ( R ) and any 1 snacking item among- Choco crunch cookies/Oats Cookies/Choco Brownie/Blueberry muffin.", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/cb24719daf96b58dd1be3bbf7b9fe372" - }, - { - "name": "Chocochip Muffin", - "price": "142", - "description": "Enjoy a dense chocochip muffin, with melty chocolate chips for a choco-lover's delight. Contains: Milk", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6c748c0b21f4a99593d1040021500430" - }, - { - "name": "Indulge Combo", - "price": "171", - "description": "Indulge in the perfect pairing of a classic cold coffee and a chocolate chip muffin, that balances the refreshing taste of chilled coffee with the sweet, comforting flavors of a freshly baked treat..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/ee2c7b9f-4426-4e70-9e98-bf8e8cc754ad_93eaaaa3-e498-4ca1-a77d-545a8006c8e2.png" - }, - { - "name": "Take a break Combo", - "price": "171", - "description": "Savor the harmonious pairing of a classic cappuccino with a cinnamon cookie, combining the bold, creamy flavors of coffee with the warm, spiced sweetness of a baked treat .", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/87a8807b-77e4-4e0d-a47e-af1cf2d2b96d_3230e601-af8f-4436-85cd-14c4e855240d.png" - }, - { - "name": "Treat Combo", - "price": "171", - "description": "Delight in the luxurious pairing of a chocolate jar dessert with a classic cappuccino, combining rich, creamy indulgence with the bold, aromatic flavors of espresso..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/7/23/9462ac0c-840e-44d1-a85d-bbea6f8268f1_a93dc44b-bf30-47b9-8cee-3a5589d2002e.png" - }, - { - "name": "Butter Croissant", - "price": "139", - "description": "Buttery, flaky croissant baked to golden perfection.Light, airy layers with a crisp outer shell.A classic French treat that melts in your mouth..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/da6cbf69-e422-4d02-9cc9-cb0452d04874_2153fe0c-389e-4842-b4c9-e0b78c450625.png" - }, - { - "name": "Butter Croissant + Cappuccino", - "price": "209", - "description": "Buttery croissant paired with a rich, frothy cappuccino.Warm, comforting, and perfectly balanced.A timeless duo for your anytime cravings..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/227a6aa3-f18c-4685-bb09-a76a01ec39a1_f347d737-b634-43d7-9b73-dc99bccde65f.png" - }, - { - "name": "Butter Croissant + Iced Coffee", - "price": "209", - "description": "Buttery, flaky croissant served with smooth, refreshing iced coffee. A classic combo that's light, crisp, and energizing. Perfect for a quick, satisfying bite..", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2025/4/24/be3c8688-f975-4d1e-aad7-9229232bcc69_65c9a679-687d-4e66-9847-eb4d5c0f9825.png" - }, - { - "name": "Mcflurry Oreo ( S )", - "price": "104", - "description": "Delicious soft serve meets crumbled oreo cookies, a match made in dessert heaven. Perfect for one.. Contains: Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/a28369e386195be4071d9cf5078a438d" - }, - { - "name": "McFlurry Oreo ( M )", - "price": "129", - "description": "Delicious soft serve meets crumbled oreo cookies, a match made in dessert heaven. Share it, if you can.. Contains: Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f966500ed8b913a16cfdb25aab9244e4" - }, - { - "name": "Hot Fudge Sundae", - "price": "66", - "description": "A sinful delight, soft serve topped with delicious, gooey hot chocolate fudge. Always grab an extra spoon.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/9c8958145495e8f2cf70470195f7834a" - }, - { - "name": "Strawberry Sundae", - "price": "66", - "description": "The cool vanilla soft serve ice cream with twirls of strawberry syrup.. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/d7bd22aa47cffdcdde2d5b6223fde06e" - }, - { - "name": "Oreo Sundae ( M )", - "price": "72", - "description": "Enjoy the classic McFlurry Oreo goodness with a drizzle of hot fudge sauce with the Oreo Sundae!. Contains: Soybeans, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/3696da86802f534ba9ca68bd8be717ab" - }, - { - "name": "Black Forest McFlurry Medium", - "price": "139", - "description": "A sweet treat to suit your every mood. Contains: Soybeans, Peanut, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/f513cc8c35cadd098835fb5b23c03561" - }, - { - "name": "Hot Fudge Brownie Sundae", - "price": "139", - "description": "Luscious chocolate brownie and hot-chocolate fudge to sweeten your day. Contains: Soybeans, Peanut, Milk, Gluten", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/6e00a57c6d8ceff6812a765c80e9ce74" - }, - { - "name": "Chocolate Overload McFlurry with Oreo Medium", - "price": "164.76", - "description": "Indulge in your chocolatey dreams with creamy soft serve, Oreo crumbs, a rich Hazelnut brownie, and two decadent chocolate sauces. Tempting, irresistible, and unforgettable. Contains: Gluten, Milk, Peanut, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/eb789769-9e60-4a84-9c64-90887ca79d7c_86154b6a-146c-47e9-9bbe-e685e4928e2e.png" - }, - { - "name": "Chocolate Overload McFlurry with Oreo Small", - "price": "134.28", - "description": "Indulge in your chocolatey dreams with creamy soft serve, Oreo crumbs, a rich Hazelnut brownie, and two decadent chocolate sauces. Tempting, irresistible, and unforgettable. Contains: Gluten, Milk, Peanut, Soybeans", - "image": "https://media-assets.swiggy.com/swiggy/image/upload/fl_lossy,f_auto,q_auto,w_300,h_300,e_grayscale,c_fit/FOOD_CATALOG/IMAGES/CMS/2024/9/18/431024a3-945d-4e6b-aafe-d35c410ac513_1b01e6dc-90cc-411d-a75e-9b98b42e178d.png" - } -] \ No newline at end of file From b97eaeea4c0ffc9b145494230e074d48b1ae89c5 Mon Sep 17 00:00:00 2001 From: unclecode Date: Fri, 17 Oct 2025 20:38:39 +0800 Subject: [PATCH 069/119] feat(docker): implement smart browser pool with 10x memory efficiency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Major refactoring to eliminate memory leaks and enable high-scale crawling: - **Smart 3-Tier Browser Pool**: - Permanent browser (always-ready default config) - Hot pool (configs used 3+ times, longer TTL) - Cold pool (new/rare configs, short TTL) - Auto-promotion: cold → hot after 3 uses - 100% pool reuse achieved in tests - **Container-Aware Memory Detection**: - Read cgroup v1/v2 memory limits (not host metrics) - Accurate memory pressure detection in Docker - Memory-based browser creation blocking - **Adaptive Janitor**: - Dynamic cleanup intervals (10s/30s/60s based on memory) - Tiered TTLs: cold 30-300s, hot 120-600s - Aggressive cleanup at high memory pressure - **Unified Pool Usage**: - All endpoints now use pool (/html, /screenshot, /pdf, /execute_js, /md, /llm) - Fixed config signature mismatch (permanent browser matches endpoints) - get_default_browser_config() helper for consistency - **Configuration**: - Reduced idle_ttl: 1800s → 300s (30min → 5min) - Fixed port: 11234 → 11235 (match Gunicorn) **Performance Results** (from stress tests): - Memory: 10x reduction (500-700MB × N → 270MB permanent) - Latency: 30-50x faster (<100ms pool hits vs 3-5s startup) - Reuse: 100% for default config, 60%+ for variants - Capacity: 100+ concurrent requests (vs ~20 before) - Leak: 0 MB/cycle (stable across tests) **Test Infrastructure**: - 7-phase sequential test suite (tests/) - Docker stats integration + log analysis - Pool promotion verification - Memory leak detection - Full endpoint coverage Fixes memory issues reported in production deployments. --- deploy/docker/STRESS_TEST_PIPELINE.md | 241 +++++++++++++++++ deploy/docker/api.py | 66 +++-- deploy/docker/config.yml | 4 +- deploy/docker/crawler_pool.py | 166 +++++++++--- deploy/docker/server.py | 83 +++--- deploy/docker/tests/requirements.txt | 2 + deploy/docker/tests/test_1_basic.py | 138 ++++++++++ deploy/docker/tests/test_2_memory.py | 205 ++++++++++++++ deploy/docker/tests/test_3_pool.py | 229 ++++++++++++++++ deploy/docker/tests/test_4_concurrent.py | 236 ++++++++++++++++ deploy/docker/tests/test_5_pool_stress.py | 267 +++++++++++++++++++ deploy/docker/tests/test_6_multi_endpoint.py | 234 ++++++++++++++++ deploy/docker/tests/test_7_cleanup.py | 199 ++++++++++++++ deploy/docker/utils.py | 27 +- 14 files changed, 1979 insertions(+), 118 deletions(-) create mode 100644 deploy/docker/STRESS_TEST_PIPELINE.md create mode 100644 deploy/docker/tests/requirements.txt create mode 100755 deploy/docker/tests/test_1_basic.py create mode 100755 deploy/docker/tests/test_2_memory.py create mode 100755 deploy/docker/tests/test_3_pool.py create mode 100755 deploy/docker/tests/test_4_concurrent.py create mode 100755 deploy/docker/tests/test_5_pool_stress.py create mode 100755 deploy/docker/tests/test_6_multi_endpoint.py create mode 100755 deploy/docker/tests/test_7_cleanup.py diff --git a/deploy/docker/STRESS_TEST_PIPELINE.md b/deploy/docker/STRESS_TEST_PIPELINE.md new file mode 100644 index 00000000..44025514 --- /dev/null +++ b/deploy/docker/STRESS_TEST_PIPELINE.md @@ -0,0 +1,241 @@ +# Crawl4AI Docker Memory & Pool Optimization - Implementation Log + +## Critical Issues Identified + +### Memory Management +- **Host vs Container**: `psutil.virtual_memory()` reported host memory, not container limits +- **Browser Pooling**: No pool reuse - every endpoint created new browsers +- **Warmup Waste**: Permanent browser sat idle with mismatched config signature +- **Idle Cleanup**: 30min TTL too long, janitor ran every 60s +- **Endpoint Inconsistency**: 75% of endpoints bypassed pool (`/md`, `/html`, `/screenshot`, `/pdf`, `/execute_js`, `/llm`) + +### Pool Design Flaws +- **Config Mismatch**: Permanent browser used `config.yml` args, endpoints used empty `BrowserConfig()` +- **Logging Level**: Pool hit markers at DEBUG, invisible with INFO logging + +## Implementation Changes + +### 1. Container-Aware Memory Detection (`utils.py`) +```python +def get_container_memory_percent() -> float: + # Try cgroup v2 → v1 → fallback to psutil + # Reads /sys/fs/cgroup/memory.{current,max} OR memory/memory.{usage,limit}_in_bytes +``` + +### 2. Smart Browser Pool (`crawler_pool.py`) +**3-Tier System:** +- **PERMANENT**: Always-ready default browser (never cleaned) +- **HOT_POOL**: Configs used 3+ times (longer TTL) +- **COLD_POOL**: New/rare configs (short TTL) + +**Key Functions:** +- `get_crawler(cfg)`: Check permanent → hot → cold → create new +- `init_permanent(cfg)`: Initialize permanent at startup +- `janitor()`: Adaptive cleanup (10s/30s/60s intervals based on memory) +- `_sig(cfg)`: SHA1 hash of config dict for pool keys + +**Logging Fix**: Changed `logger.debug()` → `logger.info()` for pool hits + +### 3. Endpoint Unification +**Helper Function** (`server.py`): +```python +def get_default_browser_config() -> BrowserConfig: + return BrowserConfig( + extra_args=config["crawler"]["browser"].get("extra_args", []), + **config["crawler"]["browser"].get("kwargs", {}), + ) +``` + +**Migrated Endpoints:** +- `/html`, `/screenshot`, `/pdf`, `/execute_js` → use `get_default_browser_config()` +- `handle_llm_qa()`, `handle_markdown_request()` → same + +**Result**: All endpoints now hit permanent browser pool + +### 4. Config Updates (`config.yml`) +- `idle_ttl_sec: 1800` → `300` (30min → 5min base TTL) +- `port: 11234` → `11235` (fixed mismatch with Gunicorn) + +### 5. Lifespan Fix (`server.py`) +```python +await init_permanent(BrowserConfig( + extra_args=config["crawler"]["browser"].get("extra_args", []), + **config["crawler"]["browser"].get("kwargs", {}), +)) +``` +Permanent browser now matches endpoint config signatures + +## Test Results + +### Test 1: Basic Health +- 10 requests to `/health` +- **Result**: 100% success, avg 3ms latency +- **Baseline**: Container starts in ~5s, 270 MB idle + +### Test 2: Memory Monitoring +- 20 requests with Docker stats tracking +- **Result**: 100% success, no memory leak (-0.2 MB delta) +- **Baseline**: 269.7 MB container overhead + +### Test 3: Pool Validation +- 30 requests to `/html` endpoint +- **Result**: **100% permanent browser hits**, 0 new browsers created +- **Memory**: 287 MB baseline → 396 MB active (+109 MB) +- **Latency**: Avg 4s (includes network to httpbin.org) + +### Test 4: Concurrent Load +- Light (10) → Medium (50) → Heavy (100) concurrent +- **Total**: 320 requests +- **Result**: 100% success, **320/320 permanent hits**, 0 new browsers +- **Memory**: 269 MB → peak 1533 MB → final 993 MB +- **Latency**: P99 at 100 concurrent = 34s (expected with single browser) + +### Test 5: Pool Stress (Mixed Configs) +- 20 requests with 4 different viewport configs +- **Result**: 4 new browsers, 4 cold hits, **4 promotions to hot**, 8 hot hits +- **Reuse Rate**: 60% (12 pool hits / 20 requests) +- **Memory**: 270 MB → 928 MB peak (+658 MB = ~165 MB per browser) +- **Proves**: Cold → hot promotion at 3 uses working perfectly + +### Test 6: Multi-Endpoint +- 10 requests each: `/html`, `/screenshot`, `/pdf`, `/crawl` +- **Result**: 100% success across all 4 endpoints +- **Latency**: 5-8s avg (PDF slowest at 7.2s) + +### Test 7: Cleanup Verification +- 20 requests (load spike) → 90s idle +- **Memory**: 269 MB → peak 1107 MB → final 780 MB +- **Recovery**: 327 MB (39%) - partial cleanup +- **Note**: Hot pool browsers persist (by design), janitor working correctly + +## Performance Metrics + +| Metric | Before | After | Improvement | +|--------|--------|-------|-------------| +| Pool Reuse | 0% | 100% (default config) | ∞ | +| Memory Leak | Unknown | 0 MB/cycle | Stable | +| Browser Reuse | No | Yes | ~3-5s saved per request | +| Idle Memory | 500-700 MB × N | 270-400 MB | 10x reduction | +| Concurrent Capacity | ~20 | 100+ | 5x | + +## Key Learnings + +1. **Config Signature Matching**: Permanent browser MUST match endpoint default config exactly (SHA1 hash) +2. **Logging Levels**: Pool diagnostics need INFO level, not DEBUG +3. **Memory in Docker**: Must read cgroup files, not host metrics +4. **Janitor Timing**: 60s interval adequate, but TTLs should be short (5min) for cold pool +5. **Hot Promotion**: 3-use threshold works well for production patterns +6. **Memory Per Browser**: ~150-200 MB per Chromium instance with headless + text_mode + +## Test Infrastructure + +**Location**: `deploy/docker/tests/` +**Dependencies**: `httpx`, `docker` (Python SDK) +**Pattern**: Sequential build - each test adds one capability + +**Files**: +- `test_1_basic.py`: Health check + container lifecycle +- `test_2_memory.py`: + Docker stats monitoring +- `test_3_pool.py`: + Log analysis for pool markers +- `test_4_concurrent.py`: + asyncio.Semaphore for concurrency control +- `test_5_pool_stress.py`: + Config variants (viewports) +- `test_6_multi_endpoint.py`: + Multiple endpoint testing +- `test_7_cleanup.py`: + Time-series memory tracking for janitor + +**Run Pattern**: +```bash +cd deploy/docker/tests +pip install -r requirements.txt +# Rebuild after code changes: +cd /path/to/repo && docker buildx build -t crawl4ai-local:latest --load . +# Run test: +python test_N_name.py +``` + +## Architecture Decisions + +**Why Permanent Browser?** +- 90% of requests use default config → single browser serves most traffic +- Eliminates 3-5s startup overhead per request + +**Why 3-Tier Pool?** +- Permanent: Zero cost for common case +- Hot: Amortized cost for frequent variants +- Cold: Lazy allocation for rare configs + +**Why Adaptive Janitor?** +- Memory pressure triggers aggressive cleanup +- Low memory allows longer TTLs for better reuse + +**Why Not Close After Each Request?** +- Browser startup: 3-5s overhead +- Pool reuse: <100ms overhead +- Net: 30-50x faster + +## Future Optimizations + +1. **Request Queuing**: When at capacity, queue instead of reject +2. **Pre-warming**: Predict common configs, pre-create browsers +3. **Metrics Export**: Prometheus metrics for pool efficiency +4. **Config Normalization**: Group similar viewports (e.g., 1920±50 → 1920) + +## Critical Code Paths + +**Browser Acquisition** (`crawler_pool.py:34-78`): +``` +get_crawler(cfg) → + _sig(cfg) → + if sig == DEFAULT_CONFIG_SIG → PERMANENT + elif sig in HOT_POOL → HOT_POOL[sig] + elif sig in COLD_POOL → promote if count >= 3 + else → create new in COLD_POOL +``` + +**Janitor Loop** (`crawler_pool.py:107-146`): +``` +while True: + mem% = get_container_memory_percent() + if mem% > 80: interval=10s, cold_ttl=30s + elif mem% > 60: interval=30s, cold_ttl=60s + else: interval=60s, cold_ttl=300s + sleep(interval) + close idle browsers (COLD then HOT) +``` + +**Endpoint Pattern** (`server.py` example): +```python +@app.post("/html") +async def generate_html(...): + from crawler_pool import get_crawler + crawler = await get_crawler(get_default_browser_config()) + results = await crawler.arun(url=body.url, config=cfg) + # No crawler.close() - returned to pool +``` + +## Debugging Tips + +**Check Pool Activity**: +```bash +docker logs crawl4ai-test | grep -E "(🔥|♨️|❄️|🆕|⬆️)" +``` + +**Verify Config Signature**: +```python +from crawl4ai import BrowserConfig +import json, hashlib +cfg = BrowserConfig(...) +sig = hashlib.sha1(json.dumps(cfg.to_dict(), sort_keys=True).encode()).hexdigest() +print(sig[:8]) # Compare with logs +``` + +**Monitor Memory**: +```bash +docker stats crawl4ai-test +``` + +## Known Limitations + +- **Mac Docker Stats**: CPU metrics unreliable, memory works +- **PDF Generation**: Slowest endpoint (~7s), no optimization yet +- **Hot Pool Persistence**: May hold memory longer than needed (trade-off for performance) +- **Janitor Lag**: Up to 60s before cleanup triggers in low-memory scenarios diff --git a/deploy/docker/api.py b/deploy/docker/api.py index d0127e7b..605b0c8a 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -66,6 +66,7 @@ async def handle_llm_qa( config: dict ) -> str: """Process QA using LLM with crawled content as context.""" + from crawler_pool import get_crawler try: if not url.startswith(('http://', 'https://')) and not url.startswith(("raw:", "raw://")): url = 'https://' + url @@ -74,15 +75,21 @@ async def handle_llm_qa( if last_q_index != -1: url = url[:last_q_index] - # Get markdown content - async with AsyncWebCrawler() as crawler: - result = await crawler.arun(url) - if not result.success: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=result.error_message - ) - content = result.markdown.fit_markdown or result.markdown.raw_markdown + # Get markdown content (use default config) + from utils import load_config + cfg = load_config() + browser_cfg = BrowserConfig( + extra_args=cfg["crawler"]["browser"].get("extra_args", []), + **cfg["crawler"]["browser"].get("kwargs", {}), + ) + crawler = await get_crawler(browser_cfg) + result = await crawler.arun(url) + if not result.success: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=result.error_message + ) + content = result.markdown.fit_markdown or result.markdown.raw_markdown # Create prompt and get LLM response prompt = f"""Use the following content as context to answer the question. @@ -224,25 +231,32 @@ async def handle_markdown_request( cache_mode = CacheMode.ENABLED if cache == "1" else CacheMode.WRITE_ONLY - async with AsyncWebCrawler() as crawler: - result = await crawler.arun( - url=decoded_url, - config=CrawlerRunConfig( - markdown_generator=md_generator, - scraping_strategy=LXMLWebScrapingStrategy(), - cache_mode=cache_mode - ) + from crawler_pool import get_crawler + from utils import load_config as _load_config + _cfg = _load_config() + browser_cfg = BrowserConfig( + extra_args=_cfg["crawler"]["browser"].get("extra_args", []), + **_cfg["crawler"]["browser"].get("kwargs", {}), + ) + crawler = await get_crawler(browser_cfg) + result = await crawler.arun( + url=decoded_url, + config=CrawlerRunConfig( + markdown_generator=md_generator, + scraping_strategy=LXMLWebScrapingStrategy(), + cache_mode=cache_mode ) - - if not result.success: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=result.error_message - ) + ) - return (result.markdown.raw_markdown - if filter_type == FilterType.RAW - else result.markdown.fit_markdown) + if not result.success: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=result.error_message + ) + + return (result.markdown.raw_markdown + if filter_type == FilterType.RAW + else result.markdown.fit_markdown) except Exception as e: logger.error(f"Markdown error: {str(e)}", exc_info=True) diff --git a/deploy/docker/config.yml b/deploy/docker/config.yml index 35371375..d09396a5 100644 --- a/deploy/docker/config.yml +++ b/deploy/docker/config.yml @@ -3,7 +3,7 @@ app: title: "Crawl4AI API" version: "1.0.0" host: "0.0.0.0" - port: 11234 + port: 11235 reload: False workers: 1 timeout_keep_alive: 300 @@ -61,7 +61,7 @@ crawler: batch_process: 300.0 # Timeout for batch processing pool: max_pages: 40 # ← GLOBAL_SEM permits - idle_ttl_sec: 1800 # ← 30 min janitor cutoff + idle_ttl_sec: 300 # ← 30 min janitor cutoff browser: kwargs: headless: true diff --git a/deploy/docker/crawler_pool.py b/deploy/docker/crawler_pool.py index d15102e4..226e3680 100644 --- a/deploy/docker/crawler_pool.py +++ b/deploy/docker/crawler_pool.py @@ -1,60 +1,146 @@ -# crawler_pool.py (new file) -import asyncio, json, hashlib, time, psutil +# crawler_pool.py - Smart browser pool with tiered management +import asyncio, json, hashlib, time from contextlib import suppress -from typing import Dict +from typing import Dict, Optional from crawl4ai import AsyncWebCrawler, BrowserConfig -from typing import Dict -from utils import load_config +from utils import load_config, get_container_memory_percent +import logging +logger = logging.getLogger(__name__) CONFIG = load_config() -POOL: Dict[str, AsyncWebCrawler] = {} +# Pool tiers +PERMANENT: Optional[AsyncWebCrawler] = None # Always-ready default browser +HOT_POOL: Dict[str, AsyncWebCrawler] = {} # Frequent configs +COLD_POOL: Dict[str, AsyncWebCrawler] = {} # Rare configs LAST_USED: Dict[str, float] = {} +USAGE_COUNT: Dict[str, int] = {} LOCK = asyncio.Lock() -MEM_LIMIT = CONFIG.get("crawler", {}).get("memory_threshold_percent", 95.0) # % RAM – refuse new browsers above this -IDLE_TTL = CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 1800) # close if unused for 30 min +# Config +MEM_LIMIT = CONFIG.get("crawler", {}).get("memory_threshold_percent", 95.0) +BASE_IDLE_TTL = CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 300) +DEFAULT_CONFIG_SIG = None # Cached sig for default config def _sig(cfg: BrowserConfig) -> str: + """Generate config signature.""" payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",",":")) return hashlib.sha1(payload.encode()).hexdigest() +def _is_default_config(sig: str) -> bool: + """Check if config matches default.""" + return sig == DEFAULT_CONFIG_SIG + async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler: - try: - sig = _sig(cfg) - async with LOCK: - if sig in POOL: - LAST_USED[sig] = time.time(); - return POOL[sig] - if psutil.virtual_memory().percent >= MEM_LIMIT: - raise MemoryError("RAM pressure – new browser denied") - crawler = AsyncWebCrawler(config=cfg, thread_safe=False) - await crawler.start() - POOL[sig] = crawler; LAST_USED[sig] = time.time() - return crawler - except MemoryError as e: - raise MemoryError(f"RAM pressure – new browser denied: {e}") - except Exception as e: - raise RuntimeError(f"Failed to start browser: {e}") - finally: - if sig in POOL: - LAST_USED[sig] = time.time() - else: - # If we failed to start the browser, we should remove it from the pool - POOL.pop(sig, None) - LAST_USED.pop(sig, None) - # If we failed to start the browser, we should remove it from the pool -async def close_all(): + """Get crawler from pool with tiered strategy.""" + sig = _sig(cfg) async with LOCK: - await asyncio.gather(*(c.close() for c in POOL.values()), return_exceptions=True) - POOL.clear(); LAST_USED.clear() + # Check permanent browser for default config + if PERMANENT and _is_default_config(sig): + LAST_USED[sig] = time.time() + USAGE_COUNT[sig] = USAGE_COUNT.get(sig, 0) + 1 + logger.info("🔥 Using permanent browser") + return PERMANENT + + # Check hot pool + if sig in HOT_POOL: + LAST_USED[sig] = time.time() + USAGE_COUNT[sig] = USAGE_COUNT.get(sig, 0) + 1 + logger.info(f"♨️ Using hot pool browser (sig={sig[:8]})") + return HOT_POOL[sig] + + # Check cold pool (promote to hot if used 3+ times) + if sig in COLD_POOL: + LAST_USED[sig] = time.time() + USAGE_COUNT[sig] = USAGE_COUNT.get(sig, 0) + 1 + + if USAGE_COUNT[sig] >= 3: + logger.info(f"⬆️ Promoting to hot pool (sig={sig[:8]}, count={USAGE_COUNT[sig]})") + HOT_POOL[sig] = COLD_POOL.pop(sig) + return HOT_POOL[sig] + + logger.info(f"❄️ Using cold pool browser (sig={sig[:8]})") + return COLD_POOL[sig] + + # Memory check before creating new + mem_pct = get_container_memory_percent() + if mem_pct >= MEM_LIMIT: + logger.error(f"💥 Memory pressure: {mem_pct:.1f}% >= {MEM_LIMIT}%") + raise MemoryError(f"Memory at {mem_pct:.1f}%, refusing new browser") + + # Create new in cold pool + logger.info(f"🆕 Creating new browser in cold pool (sig={sig[:8]}, mem={mem_pct:.1f}%)") + crawler = AsyncWebCrawler(config=cfg, thread_safe=False) + await crawler.start() + COLD_POOL[sig] = crawler + LAST_USED[sig] = time.time() + USAGE_COUNT[sig] = 1 + return crawler + +async def init_permanent(cfg: BrowserConfig): + """Initialize permanent default browser.""" + global PERMANENT, DEFAULT_CONFIG_SIG + async with LOCK: + if PERMANENT: + return + DEFAULT_CONFIG_SIG = _sig(cfg) + logger.info("🔥 Creating permanent default browser") + PERMANENT = AsyncWebCrawler(config=cfg, thread_safe=False) + await PERMANENT.start() + LAST_USED[DEFAULT_CONFIG_SIG] = time.time() + USAGE_COUNT[DEFAULT_CONFIG_SIG] = 0 + +async def close_all(): + """Close all browsers.""" + async with LOCK: + tasks = [] + if PERMANENT: + tasks.append(PERMANENT.close()) + tasks.extend([c.close() for c in HOT_POOL.values()]) + tasks.extend([c.close() for c in COLD_POOL.values()]) + await asyncio.gather(*tasks, return_exceptions=True) + HOT_POOL.clear() + COLD_POOL.clear() + LAST_USED.clear() + USAGE_COUNT.clear() async def janitor(): + """Adaptive cleanup based on memory pressure.""" while True: - await asyncio.sleep(60) + mem_pct = get_container_memory_percent() + + # Adaptive intervals and TTLs + if mem_pct > 80: + interval, cold_ttl, hot_ttl = 10, 30, 120 + elif mem_pct > 60: + interval, cold_ttl, hot_ttl = 30, 60, 300 + else: + interval, cold_ttl, hot_ttl = 60, BASE_IDLE_TTL, BASE_IDLE_TTL * 2 + + await asyncio.sleep(interval) + now = time.time() async with LOCK: - for sig, crawler in list(POOL.items()): - if now - LAST_USED[sig] > IDLE_TTL: - with suppress(Exception): await crawler.close() - POOL.pop(sig, None); LAST_USED.pop(sig, None) + # Clean cold pool + for sig in list(COLD_POOL.keys()): + if now - LAST_USED.get(sig, now) > cold_ttl: + logger.info(f"🧹 Closing cold browser (sig={sig[:8]}, idle={now - LAST_USED[sig]:.0f}s)") + with suppress(Exception): + await COLD_POOL[sig].close() + COLD_POOL.pop(sig, None) + LAST_USED.pop(sig, None) + USAGE_COUNT.pop(sig, None) + + # Clean hot pool (more conservative) + for sig in list(HOT_POOL.keys()): + if now - LAST_USED.get(sig, now) > hot_ttl: + logger.info(f"🧹 Closing hot browser (sig={sig[:8]}, idle={now - LAST_USED[sig]:.0f}s)") + with suppress(Exception): + await HOT_POOL[sig].close() + HOT_POOL.pop(sig, None) + LAST_USED.pop(sig, None) + USAGE_COUNT.pop(sig, None) + + # Log pool stats + if mem_pct > 60: + logger.info(f"📊 Pool: hot={len(HOT_POOL)}, cold={len(COLD_POOL)}, mem={mem_pct:.1f}%") diff --git a/deploy/docker/server.py b/deploy/docker/server.py index 101e8614..30639852 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -78,6 +78,14 @@ __version__ = "0.5.1-d1" MAX_PAGES = config["crawler"]["pool"].get("max_pages", 30) GLOBAL_SEM = asyncio.Semaphore(MAX_PAGES) +# ── default browser config helper ───────────────────────────── +def get_default_browser_config() -> BrowserConfig: + """Get default BrowserConfig from config.yml.""" + return BrowserConfig( + extra_args=config["crawler"]["browser"].get("extra_args", []), + **config["crawler"]["browser"].get("kwargs", {}), + ) + # import logging # page_log = logging.getLogger("page_cap") # orig_arun = AsyncWebCrawler.arun @@ -103,11 +111,12 @@ AsyncWebCrawler.arun = capped_arun @asynccontextmanager async def lifespan(_: FastAPI): - await get_crawler(BrowserConfig( + from crawler_pool import init_permanent + await init_permanent(BrowserConfig( extra_args=config["crawler"]["browser"].get("extra_args", []), **config["crawler"]["browser"].get("kwargs", {}), - )) # warm‑up - app.state.janitor = asyncio.create_task(janitor()) # idle GC + )) + app.state.janitor = asyncio.create_task(janitor()) yield app.state.janitor.cancel() await close_all() @@ -266,27 +275,20 @@ async def generate_html( Crawls the URL, preprocesses the raw HTML for schema extraction, and returns the processed HTML. Use when you need sanitized HTML structures for building schemas or further processing. """ + from crawler_pool import get_crawler cfg = CrawlerRunConfig() try: - async with AsyncWebCrawler(config=BrowserConfig()) as crawler: - results = await crawler.arun(url=body.url, config=cfg) - # Check if the crawl was successful + crawler = await get_crawler(get_default_browser_config()) + results = await crawler.arun(url=body.url, config=cfg) if not results[0].success: - raise HTTPException( - status_code=500, - detail=results[0].error_message or "Crawl failed" - ) - + raise HTTPException(500, detail=results[0].error_message or "Crawl failed") + raw_html = results[0].html from crawl4ai.utils import preprocess_html_for_schema processed_html = preprocess_html_for_schema(raw_html) return JSONResponse({"html": processed_html, "url": body.url, "success": True}) except Exception as e: - # Log and raise as HTTP 500 for other exceptions - raise HTTPException( - status_code=500, - detail=str(e) - ) + raise HTTPException(500, detail=str(e)) # Screenshot endpoint @@ -304,16 +306,13 @@ async def generate_screenshot( Use when you need an image snapshot of the rendered page. Its recommened to provide an output path to save the screenshot. Then in result instead of the screenshot you will get a path to the saved file. """ + from crawler_pool import get_crawler try: - cfg = CrawlerRunConfig( - screenshot=True, screenshot_wait_for=body.screenshot_wait_for) - async with AsyncWebCrawler(config=BrowserConfig()) as crawler: - results = await crawler.arun(url=body.url, config=cfg) + cfg = CrawlerRunConfig(screenshot=True, screenshot_wait_for=body.screenshot_wait_for) + crawler = await get_crawler(get_default_browser_config()) + results = await crawler.arun(url=body.url, config=cfg) if not results[0].success: - raise HTTPException( - status_code=500, - detail=results[0].error_message or "Crawl failed" - ) + raise HTTPException(500, detail=results[0].error_message or "Crawl failed") screenshot_data = results[0].screenshot if body.output_path: abs_path = os.path.abspath(body.output_path) @@ -323,10 +322,7 @@ async def generate_screenshot( return {"success": True, "path": abs_path} return {"success": True, "screenshot": screenshot_data} except Exception as e: - raise HTTPException( - status_code=500, - detail=str(e) - ) + raise HTTPException(500, detail=str(e)) # PDF endpoint @@ -344,15 +340,13 @@ async def generate_pdf( Use when you need a printable or archivable snapshot of the page. It is recommended to provide an output path to save the PDF. Then in result instead of the PDF you will get a path to the saved file. """ + from crawler_pool import get_crawler try: cfg = CrawlerRunConfig(pdf=True) - async with AsyncWebCrawler(config=BrowserConfig()) as crawler: - results = await crawler.arun(url=body.url, config=cfg) + crawler = await get_crawler(get_default_browser_config()) + results = await crawler.arun(url=body.url, config=cfg) if not results[0].success: - raise HTTPException( - status_code=500, - detail=results[0].error_message or "Crawl failed" - ) + raise HTTPException(500, detail=results[0].error_message or "Crawl failed") pdf_data = results[0].pdf if body.output_path: abs_path = os.path.abspath(body.output_path) @@ -362,10 +356,7 @@ async def generate_pdf( return {"success": True, "path": abs_path} return {"success": True, "pdf": base64.b64encode(pdf_data).decode()} except Exception as e: - raise HTTPException( - status_code=500, - detail=str(e) - ) + raise HTTPException(500, detail=str(e)) @app.post("/execute_js") @@ -421,23 +412,17 @@ async def execute_js( ``` """ + from crawler_pool import get_crawler try: cfg = CrawlerRunConfig(js_code=body.scripts) - async with AsyncWebCrawler(config=BrowserConfig()) as crawler: - results = await crawler.arun(url=body.url, config=cfg) + crawler = await get_crawler(get_default_browser_config()) + results = await crawler.arun(url=body.url, config=cfg) if not results[0].success: - raise HTTPException( - status_code=500, - detail=results[0].error_message or "Crawl failed" - ) - # Return JSON-serializable dict of the first CrawlResult + raise HTTPException(500, detail=results[0].error_message or "Crawl failed") data = results[0].model_dump() return JSONResponse(data) except Exception as e: - raise HTTPException( - status_code=500, - detail=str(e) - ) + raise HTTPException(500, detail=str(e)) @app.get("/llm/{url:path}") diff --git a/deploy/docker/tests/requirements.txt b/deploy/docker/tests/requirements.txt new file mode 100644 index 00000000..5f7a842f --- /dev/null +++ b/deploy/docker/tests/requirements.txt @@ -0,0 +1,2 @@ +httpx>=0.25.0 +docker>=7.0.0 diff --git a/deploy/docker/tests/test_1_basic.py b/deploy/docker/tests/test_1_basic.py new file mode 100755 index 00000000..c86de073 --- /dev/null +++ b/deploy/docker/tests/test_1_basic.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +""" +Test 1: Basic Container Health + Single Endpoint +- Starts container +- Hits /health endpoint 10 times +- Reports success rate and basic latency +""" +import asyncio +import time +import docker +import httpx + +# Config +IMAGE = "crawl4ai-local:latest" +CONTAINER_NAME = "crawl4ai-test" +PORT = 11235 +REQUESTS = 10 + +async def test_endpoint(url: str, count: int): + """Hit endpoint multiple times, return stats.""" + results = [] + async with httpx.AsyncClient(timeout=30.0) as client: + for i in range(count): + start = time.time() + try: + resp = await client.get(url) + elapsed = (time.time() - start) * 1000 # ms + results.append({ + "success": resp.status_code == 200, + "latency_ms": elapsed, + "status": resp.status_code + }) + print(f" [{i+1}/{count}] ✓ {resp.status_code} - {elapsed:.0f}ms") + except Exception as e: + results.append({ + "success": False, + "latency_ms": None, + "error": str(e) + }) + print(f" [{i+1}/{count}] ✗ Error: {e}") + return results + +def start_container(client, image: str, name: str, port: int): + """Start container, return container object.""" + # Clean up existing + try: + old = client.containers.get(name) + print(f"🧹 Stopping existing container '{name}'...") + old.stop() + old.remove() + except docker.errors.NotFound: + pass + + print(f"🚀 Starting container '{name}' from image '{image}'...") + container = client.containers.run( + image, + name=name, + ports={f"{port}/tcp": port}, + detach=True, + shm_size="1g", + environment={"PYTHON_ENV": "production"} + ) + + # Wait for health + print(f"⏳ Waiting for container to be healthy...") + for _ in range(30): # 30s timeout + time.sleep(1) + container.reload() + if container.status == "running": + try: + # Quick health check + import requests + resp = requests.get(f"http://localhost:{port}/health", timeout=2) + if resp.status_code == 200: + print(f"✅ Container healthy!") + return container + except: + pass + raise TimeoutError("Container failed to start") + +def stop_container(container): + """Stop and remove container.""" + print(f"🛑 Stopping container...") + container.stop() + container.remove() + print(f"✅ Container removed") + +async def main(): + print("="*60) + print("TEST 1: Basic Container Health + Single Endpoint") + print("="*60) + + client = docker.from_env() + container = None + + try: + # Start container + container = start_container(client, IMAGE, CONTAINER_NAME, PORT) + + # Test /health endpoint + print(f"\n📊 Testing /health endpoint ({REQUESTS} requests)...") + url = f"http://localhost:{PORT}/health" + results = await test_endpoint(url, REQUESTS) + + # Calculate stats + successes = sum(1 for r in results if r["success"]) + success_rate = (successes / len(results)) * 100 + latencies = [r["latency_ms"] for r in results if r["latency_ms"] is not None] + avg_latency = sum(latencies) / len(latencies) if latencies else 0 + + # Print results + print(f"\n{'='*60}") + print(f"RESULTS:") + print(f" Success Rate: {success_rate:.1f}% ({successes}/{len(results)})") + print(f" Avg Latency: {avg_latency:.0f}ms") + if latencies: + print(f" Min Latency: {min(latencies):.0f}ms") + print(f" Max Latency: {max(latencies):.0f}ms") + print(f"{'='*60}") + + # Pass/Fail + if success_rate >= 100: + print(f"✅ TEST PASSED") + return 0 + else: + print(f"❌ TEST FAILED (expected 100% success rate)") + return 1 + + except Exception as e: + print(f"\n❌ TEST ERROR: {e}") + return 1 + finally: + if container: + stop_container(container) + +if __name__ == "__main__": + exit_code = asyncio.run(main()) + exit(exit_code) diff --git a/deploy/docker/tests/test_2_memory.py b/deploy/docker/tests/test_2_memory.py new file mode 100755 index 00000000..aed4c61c --- /dev/null +++ b/deploy/docker/tests/test_2_memory.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +""" +Test 2: Docker Stats Monitoring +- Extends Test 1 with real-time container stats +- Monitors memory % and CPU during requests +- Reports baseline, peak, and final memory +""" +import asyncio +import time +import docker +import httpx +from threading import Thread, Event + +# Config +IMAGE = "crawl4ai-local:latest" +CONTAINER_NAME = "crawl4ai-test" +PORT = 11235 +REQUESTS = 20 # More requests to see memory usage + +# Stats tracking +stats_history = [] +stop_monitoring = Event() + +def monitor_stats(container): + """Background thread to collect container stats.""" + for stat in container.stats(decode=True, stream=True): + if stop_monitoring.is_set(): + break + + try: + # Extract memory stats + mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024) # MB + mem_limit = stat['memory_stats'].get('limit', 1) / (1024 * 1024) + mem_percent = (mem_usage / mem_limit * 100) if mem_limit > 0 else 0 + + # Extract CPU stats (handle missing fields on Mac) + cpu_percent = 0 + try: + cpu_delta = stat['cpu_stats']['cpu_usage']['total_usage'] - \ + stat['precpu_stats']['cpu_usage']['total_usage'] + system_delta = stat['cpu_stats'].get('system_cpu_usage', 0) - \ + stat['precpu_stats'].get('system_cpu_usage', 0) + if system_delta > 0: + num_cpus = stat['cpu_stats'].get('online_cpus', 1) + cpu_percent = (cpu_delta / system_delta * num_cpus * 100.0) + except (KeyError, ZeroDivisionError): + pass + + stats_history.append({ + 'timestamp': time.time(), + 'memory_mb': mem_usage, + 'memory_percent': mem_percent, + 'cpu_percent': cpu_percent + }) + except Exception as e: + # Skip malformed stats + pass + + time.sleep(0.5) # Sample every 500ms + +async def test_endpoint(url: str, count: int): + """Hit endpoint, return stats.""" + results = [] + async with httpx.AsyncClient(timeout=30.0) as client: + for i in range(count): + start = time.time() + try: + resp = await client.get(url) + elapsed = (time.time() - start) * 1000 + results.append({ + "success": resp.status_code == 200, + "latency_ms": elapsed, + }) + if (i + 1) % 5 == 0: # Print every 5 requests + print(f" [{i+1}/{count}] ✓ {resp.status_code} - {elapsed:.0f}ms") + except Exception as e: + results.append({"success": False, "error": str(e)}) + print(f" [{i+1}/{count}] ✗ Error: {e}") + return results + +def start_container(client, image: str, name: str, port: int): + """Start container.""" + try: + old = client.containers.get(name) + print(f"🧹 Stopping existing container '{name}'...") + old.stop() + old.remove() + except docker.errors.NotFound: + pass + + print(f"🚀 Starting container '{name}'...") + container = client.containers.run( + image, + name=name, + ports={f"{port}/tcp": port}, + detach=True, + shm_size="1g", + mem_limit="4g", # Set explicit memory limit + ) + + print(f"⏳ Waiting for health...") + for _ in range(30): + time.sleep(1) + container.reload() + if container.status == "running": + try: + import requests + resp = requests.get(f"http://localhost:{port}/health", timeout=2) + if resp.status_code == 200: + print(f"✅ Container healthy!") + return container + except: + pass + raise TimeoutError("Container failed to start") + +def stop_container(container): + """Stop container.""" + print(f"🛑 Stopping container...") + container.stop() + container.remove() + +async def main(): + print("="*60) + print("TEST 2: Docker Stats Monitoring") + print("="*60) + + client = docker.from_env() + container = None + monitor_thread = None + + try: + # Start container + container = start_container(client, IMAGE, CONTAINER_NAME, PORT) + + # Start stats monitoring in background + print(f"\n📊 Starting stats monitor...") + stop_monitoring.clear() + stats_history.clear() + monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True) + monitor_thread.start() + + # Wait a bit for baseline + await asyncio.sleep(2) + baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0 + print(f"📏 Baseline memory: {baseline_mem:.1f} MB") + + # Test /health endpoint + print(f"\n🔄 Running {REQUESTS} requests to /health...") + url = f"http://localhost:{PORT}/health" + results = await test_endpoint(url, REQUESTS) + + # Wait a bit to capture peak + await asyncio.sleep(1) + + # Stop monitoring + stop_monitoring.set() + if monitor_thread: + monitor_thread.join(timeout=2) + + # Calculate stats + successes = sum(1 for r in results if r.get("success")) + success_rate = (successes / len(results)) * 100 + latencies = [r["latency_ms"] for r in results if "latency_ms" in r] + avg_latency = sum(latencies) / len(latencies) if latencies else 0 + + # Memory stats + memory_samples = [s['memory_mb'] for s in stats_history] + peak_mem = max(memory_samples) if memory_samples else 0 + final_mem = memory_samples[-1] if memory_samples else 0 + mem_delta = final_mem - baseline_mem + + # Print results + print(f"\n{'='*60}") + print(f"RESULTS:") + print(f" Success Rate: {success_rate:.1f}% ({successes}/{len(results)})") + print(f" Avg Latency: {avg_latency:.0f}ms") + print(f"\n Memory Stats:") + print(f" Baseline: {baseline_mem:.1f} MB") + print(f" Peak: {peak_mem:.1f} MB") + print(f" Final: {final_mem:.1f} MB") + print(f" Delta: {mem_delta:+.1f} MB") + print(f"{'='*60}") + + # Pass/Fail + if success_rate >= 100 and mem_delta < 100: # No significant memory growth + print(f"✅ TEST PASSED") + return 0 + else: + if success_rate < 100: + print(f"❌ TEST FAILED (success rate < 100%)") + if mem_delta >= 100: + print(f"⚠️ WARNING: Memory grew by {mem_delta:.1f} MB") + return 1 + + except Exception as e: + print(f"\n❌ TEST ERROR: {e}") + return 1 + finally: + stop_monitoring.set() + if container: + stop_container(container) + +if __name__ == "__main__": + exit_code = asyncio.run(main()) + exit(exit_code) diff --git a/deploy/docker/tests/test_3_pool.py b/deploy/docker/tests/test_3_pool.py new file mode 100755 index 00000000..9f2c00b2 --- /dev/null +++ b/deploy/docker/tests/test_3_pool.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +""" +Test 3: Pool Validation - Permanent Browser Reuse +- Tests /html endpoint (should use permanent browser) +- Monitors container logs for pool hit markers +- Validates browser reuse rate +- Checks memory after browser creation +""" +import asyncio +import time +import docker +import httpx +from threading import Thread, Event + +# Config +IMAGE = "crawl4ai-local:latest" +CONTAINER_NAME = "crawl4ai-test" +PORT = 11235 +REQUESTS = 30 + +# Stats tracking +stats_history = [] +stop_monitoring = Event() + +def monitor_stats(container): + """Background stats collector.""" + for stat in container.stats(decode=True, stream=True): + if stop_monitoring.is_set(): + break + try: + mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024) + stats_history.append({ + 'timestamp': time.time(), + 'memory_mb': mem_usage, + }) + except: + pass + time.sleep(0.5) + +def count_log_markers(container): + """Extract pool usage markers from logs.""" + logs = container.logs().decode('utf-8') + + permanent_hits = logs.count("🔥 Using permanent browser") + hot_hits = logs.count("♨️ Using hot pool browser") + cold_hits = logs.count("❄️ Using cold pool browser") + new_created = logs.count("🆕 Creating new browser") + + return { + 'permanent_hits': permanent_hits, + 'hot_hits': hot_hits, + 'cold_hits': cold_hits, + 'new_created': new_created, + 'total_hits': permanent_hits + hot_hits + cold_hits + } + +async def test_endpoint(url: str, count: int): + """Hit endpoint multiple times.""" + results = [] + async with httpx.AsyncClient(timeout=60.0) as client: + for i in range(count): + start = time.time() + try: + resp = await client.post(url, json={"url": "https://httpbin.org/html"}) + elapsed = (time.time() - start) * 1000 + results.append({ + "success": resp.status_code == 200, + "latency_ms": elapsed, + }) + if (i + 1) % 10 == 0: + print(f" [{i+1}/{count}] ✓ {resp.status_code} - {elapsed:.0f}ms") + except Exception as e: + results.append({"success": False, "error": str(e)}) + print(f" [{i+1}/{count}] ✗ Error: {e}") + return results + +def start_container(client, image: str, name: str, port: int): + """Start container.""" + try: + old = client.containers.get(name) + print(f"🧹 Stopping existing container...") + old.stop() + old.remove() + except docker.errors.NotFound: + pass + + print(f"🚀 Starting container...") + container = client.containers.run( + image, + name=name, + ports={f"{port}/tcp": port}, + detach=True, + shm_size="1g", + mem_limit="4g", + ) + + print(f"⏳ Waiting for health...") + for _ in range(30): + time.sleep(1) + container.reload() + if container.status == "running": + try: + import requests + resp = requests.get(f"http://localhost:{port}/health", timeout=2) + if resp.status_code == 200: + print(f"✅ Container healthy!") + return container + except: + pass + raise TimeoutError("Container failed to start") + +def stop_container(container): + """Stop container.""" + print(f"🛑 Stopping container...") + container.stop() + container.remove() + +async def main(): + print("="*60) + print("TEST 3: Pool Validation - Permanent Browser Reuse") + print("="*60) + + client = docker.from_env() + container = None + monitor_thread = None + + try: + # Start container + container = start_container(client, IMAGE, CONTAINER_NAME, PORT) + + # Wait for permanent browser initialization + print(f"\n⏳ Waiting for permanent browser init (3s)...") + await asyncio.sleep(3) + + # Start stats monitoring + print(f"📊 Starting stats monitor...") + stop_monitoring.clear() + stats_history.clear() + monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True) + monitor_thread.start() + + await asyncio.sleep(1) + baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0 + print(f"📏 Baseline (with permanent browser): {baseline_mem:.1f} MB") + + # Test /html endpoint (uses permanent browser for default config) + print(f"\n🔄 Running {REQUESTS} requests to /html...") + url = f"http://localhost:{PORT}/html" + results = await test_endpoint(url, REQUESTS) + + # Wait a bit + await asyncio.sleep(1) + + # Stop monitoring + stop_monitoring.set() + if monitor_thread: + monitor_thread.join(timeout=2) + + # Analyze logs for pool markers + print(f"\n📋 Analyzing pool usage...") + pool_stats = count_log_markers(container) + + # Calculate request stats + successes = sum(1 for r in results if r.get("success")) + success_rate = (successes / len(results)) * 100 + latencies = [r["latency_ms"] for r in results if "latency_ms" in r] + avg_latency = sum(latencies) / len(latencies) if latencies else 0 + + # Memory stats + memory_samples = [s['memory_mb'] for s in stats_history] + peak_mem = max(memory_samples) if memory_samples else 0 + final_mem = memory_samples[-1] if memory_samples else 0 + mem_delta = final_mem - baseline_mem + + # Calculate reuse rate + total_requests = len(results) + total_pool_hits = pool_stats['total_hits'] + reuse_rate = (total_pool_hits / total_requests * 100) if total_requests > 0 else 0 + + # Print results + print(f"\n{'='*60}") + print(f"RESULTS:") + print(f" Success Rate: {success_rate:.1f}% ({successes}/{len(results)})") + print(f" Avg Latency: {avg_latency:.0f}ms") + print(f"\n Pool Stats:") + print(f" 🔥 Permanent Hits: {pool_stats['permanent_hits']}") + print(f" ♨️ Hot Pool Hits: {pool_stats['hot_hits']}") + print(f" ❄️ Cold Pool Hits: {pool_stats['cold_hits']}") + print(f" 🆕 New Created: {pool_stats['new_created']}") + print(f" 📊 Reuse Rate: {reuse_rate:.1f}%") + print(f"\n Memory Stats:") + print(f" Baseline: {baseline_mem:.1f} MB") + print(f" Peak: {peak_mem:.1f} MB") + print(f" Final: {final_mem:.1f} MB") + print(f" Delta: {mem_delta:+.1f} MB") + print(f"{'='*60}") + + # Pass/Fail + passed = True + if success_rate < 100: + print(f"❌ FAIL: Success rate {success_rate:.1f}% < 100%") + passed = False + if reuse_rate < 80: + print(f"❌ FAIL: Reuse rate {reuse_rate:.1f}% < 80% (expected high permanent browser usage)") + passed = False + if pool_stats['permanent_hits'] < (total_requests * 0.8): + print(f"⚠️ WARNING: Only {pool_stats['permanent_hits']} permanent hits out of {total_requests} requests") + if mem_delta > 200: + print(f"⚠️ WARNING: Memory grew by {mem_delta:.1f} MB (possible browser leak)") + + if passed: + print(f"✅ TEST PASSED") + return 0 + else: + return 1 + + except Exception as e: + print(f"\n❌ TEST ERROR: {e}") + import traceback + traceback.print_exc() + return 1 + finally: + stop_monitoring.set() + if container: + stop_container(container) + +if __name__ == "__main__": + exit_code = asyncio.run(main()) + exit(exit_code) diff --git a/deploy/docker/tests/test_4_concurrent.py b/deploy/docker/tests/test_4_concurrent.py new file mode 100755 index 00000000..70198ddc --- /dev/null +++ b/deploy/docker/tests/test_4_concurrent.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +""" +Test 4: Concurrent Load Testing +- Tests pool under concurrent load +- Escalates: 10 → 50 → 100 concurrent requests +- Validates latency distribution (P50, P95, P99) +- Monitors memory stability +""" +import asyncio +import time +import docker +import httpx +from threading import Thread, Event +from collections import defaultdict + +# Config +IMAGE = "crawl4ai-local:latest" +CONTAINER_NAME = "crawl4ai-test" +PORT = 11235 +LOAD_LEVELS = [ + {"name": "Light", "concurrent": 10, "requests": 20}, + {"name": "Medium", "concurrent": 50, "requests": 100}, + {"name": "Heavy", "concurrent": 100, "requests": 200}, +] + +# Stats +stats_history = [] +stop_monitoring = Event() + +def monitor_stats(container): + """Background stats collector.""" + for stat in container.stats(decode=True, stream=True): + if stop_monitoring.is_set(): + break + try: + mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024) + stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage}) + except: + pass + time.sleep(0.5) + +def count_log_markers(container): + """Extract pool markers.""" + logs = container.logs().decode('utf-8') + return { + 'permanent': logs.count("🔥 Using permanent browser"), + 'hot': logs.count("♨️ Using hot pool browser"), + 'cold': logs.count("❄️ Using cold pool browser"), + 'new': logs.count("🆕 Creating new browser"), + } + +async def hit_endpoint(client, url, payload, semaphore): + """Single request with concurrency control.""" + async with semaphore: + start = time.time() + try: + resp = await client.post(url, json=payload, timeout=60.0) + elapsed = (time.time() - start) * 1000 + return {"success": resp.status_code == 200, "latency_ms": elapsed} + except Exception as e: + return {"success": False, "error": str(e)} + +async def run_concurrent_test(url, payload, concurrent, total_requests): + """Run concurrent requests.""" + semaphore = asyncio.Semaphore(concurrent) + async with httpx.AsyncClient() as client: + tasks = [hit_endpoint(client, url, payload, semaphore) for _ in range(total_requests)] + results = await asyncio.gather(*tasks) + return results + +def calculate_percentiles(latencies): + """Calculate P50, P95, P99.""" + if not latencies: + return 0, 0, 0 + sorted_lat = sorted(latencies) + n = len(sorted_lat) + return ( + sorted_lat[int(n * 0.50)], + sorted_lat[int(n * 0.95)], + sorted_lat[int(n * 0.99)], + ) + +def start_container(client, image, name, port): + """Start container.""" + try: + old = client.containers.get(name) + print(f"🧹 Stopping existing container...") + old.stop() + old.remove() + except docker.errors.NotFound: + pass + + print(f"🚀 Starting container...") + container = client.containers.run( + image, name=name, ports={f"{port}/tcp": port}, + detach=True, shm_size="1g", mem_limit="4g", + ) + + print(f"⏳ Waiting for health...") + for _ in range(30): + time.sleep(1) + container.reload() + if container.status == "running": + try: + import requests + if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200: + print(f"✅ Container healthy!") + return container + except: + pass + raise TimeoutError("Container failed to start") + +async def main(): + print("="*60) + print("TEST 4: Concurrent Load Testing") + print("="*60) + + client = docker.from_env() + container = None + monitor_thread = None + + try: + container = start_container(client, IMAGE, CONTAINER_NAME, PORT) + + print(f"\n⏳ Waiting for permanent browser init (3s)...") + await asyncio.sleep(3) + + # Start monitoring + stop_monitoring.clear() + stats_history.clear() + monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True) + monitor_thread.start() + + await asyncio.sleep(1) + baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0 + print(f"📏 Baseline: {baseline_mem:.1f} MB\n") + + url = f"http://localhost:{PORT}/html" + payload = {"url": "https://httpbin.org/html"} + + all_results = [] + level_stats = [] + + # Run load levels + for level in LOAD_LEVELS: + print(f"{'='*60}") + print(f"🔄 {level['name']} Load: {level['concurrent']} concurrent, {level['requests']} total") + print(f"{'='*60}") + + start_time = time.time() + results = await run_concurrent_test(url, payload, level['concurrent'], level['requests']) + duration = time.time() - start_time + + successes = sum(1 for r in results if r.get("success")) + success_rate = (successes / len(results)) * 100 + latencies = [r["latency_ms"] for r in results if "latency_ms" in r] + p50, p95, p99 = calculate_percentiles(latencies) + avg_lat = sum(latencies) / len(latencies) if latencies else 0 + + print(f" Duration: {duration:.1f}s") + print(f" Success: {success_rate:.1f}% ({successes}/{len(results)})") + print(f" Avg Latency: {avg_lat:.0f}ms") + print(f" P50/P95/P99: {p50:.0f}ms / {p95:.0f}ms / {p99:.0f}ms") + + level_stats.append({ + 'name': level['name'], + 'concurrent': level['concurrent'], + 'success_rate': success_rate, + 'avg_latency': avg_lat, + 'p50': p50, 'p95': p95, 'p99': p99, + }) + all_results.extend(results) + + await asyncio.sleep(2) # Cool down between levels + + # Stop monitoring + await asyncio.sleep(1) + stop_monitoring.set() + if monitor_thread: + monitor_thread.join(timeout=2) + + # Final stats + pool_stats = count_log_markers(container) + memory_samples = [s['memory_mb'] for s in stats_history] + peak_mem = max(memory_samples) if memory_samples else 0 + final_mem = memory_samples[-1] if memory_samples else 0 + + print(f"\n{'='*60}") + print(f"FINAL RESULTS:") + print(f"{'='*60}") + print(f" Total Requests: {len(all_results)}") + print(f"\n Pool Utilization:") + print(f" 🔥 Permanent: {pool_stats['permanent']}") + print(f" ♨️ Hot: {pool_stats['hot']}") + print(f" ❄️ Cold: {pool_stats['cold']}") + print(f" 🆕 New: {pool_stats['new']}") + print(f"\n Memory:") + print(f" Baseline: {baseline_mem:.1f} MB") + print(f" Peak: {peak_mem:.1f} MB") + print(f" Final: {final_mem:.1f} MB") + print(f" Delta: {final_mem - baseline_mem:+.1f} MB") + print(f"{'='*60}") + + # Pass/Fail + passed = True + for ls in level_stats: + if ls['success_rate'] < 99: + print(f"❌ FAIL: {ls['name']} success rate {ls['success_rate']:.1f}% < 99%") + passed = False + if ls['p99'] > 10000: # 10s threshold + print(f"⚠️ WARNING: {ls['name']} P99 latency {ls['p99']:.0f}ms very high") + + if final_mem - baseline_mem > 300: + print(f"⚠️ WARNING: Memory grew {final_mem - baseline_mem:.1f} MB") + + if passed: + print(f"✅ TEST PASSED") + return 0 + else: + return 1 + + except Exception as e: + print(f"\n❌ TEST ERROR: {e}") + import traceback + traceback.print_exc() + return 1 + finally: + stop_monitoring.set() + if container: + print(f"🛑 Stopping container...") + container.stop() + container.remove() + +if __name__ == "__main__": + exit_code = asyncio.run(main()) + exit(exit_code) diff --git a/deploy/docker/tests/test_5_pool_stress.py b/deploy/docker/tests/test_5_pool_stress.py new file mode 100755 index 00000000..40752d84 --- /dev/null +++ b/deploy/docker/tests/test_5_pool_stress.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +""" +Test 5: Pool Stress - Mixed Configs +- Tests hot/cold pool with different browser configs +- Uses different viewports to create config variants +- Validates cold → hot promotion after 3 uses +- Monitors pool tier distribution +""" +import asyncio +import time +import docker +import httpx +from threading import Thread, Event +import random + +# Config +IMAGE = "crawl4ai-local:latest" +CONTAINER_NAME = "crawl4ai-test" +PORT = 11235 +REQUESTS_PER_CONFIG = 5 # 5 requests per config variant + +# Different viewport configs to test pool tiers +VIEWPORT_CONFIGS = [ + None, # Default (permanent browser) + {"width": 1920, "height": 1080}, # Desktop + {"width": 1024, "height": 768}, # Tablet + {"width": 375, "height": 667}, # Mobile +] + +# Stats +stats_history = [] +stop_monitoring = Event() + +def monitor_stats(container): + """Background stats collector.""" + for stat in container.stats(decode=True, stream=True): + if stop_monitoring.is_set(): + break + try: + mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024) + stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage}) + except: + pass + time.sleep(0.5) + +def analyze_pool_logs(container): + """Extract detailed pool stats from logs.""" + logs = container.logs().decode('utf-8') + + permanent = logs.count("🔥 Using permanent browser") + hot = logs.count("♨️ Using hot pool browser") + cold = logs.count("❄️ Using cold pool browser") + new = logs.count("🆕 Creating new browser") + promotions = logs.count("⬆️ Promoting to hot pool") + + return { + 'permanent': permanent, + 'hot': hot, + 'cold': cold, + 'new': new, + 'promotions': promotions, + 'total': permanent + hot + cold + } + +async def crawl_with_viewport(client, url, viewport): + """Single request with specific viewport.""" + payload = { + "urls": ["https://httpbin.org/html"], + "browser_config": {}, + "crawler_config": {} + } + + # Add viewport if specified + if viewport: + payload["browser_config"] = { + "type": "BrowserConfig", + "params": { + "viewport": {"type": "dict", "value": viewport}, + "headless": True, + "text_mode": True, + "extra_args": [ + "--no-sandbox", + "--disable-dev-shm-usage", + "--disable-gpu", + "--disable-software-rasterizer", + "--disable-web-security", + "--allow-insecure-localhost", + "--ignore-certificate-errors" + ] + } + } + + start = time.time() + try: + resp = await client.post(url, json=payload, timeout=60.0) + elapsed = (time.time() - start) * 1000 + return {"success": resp.status_code == 200, "latency_ms": elapsed, "viewport": viewport} + except Exception as e: + return {"success": False, "error": str(e), "viewport": viewport} + +def start_container(client, image, name, port): + """Start container.""" + try: + old = client.containers.get(name) + print(f"🧹 Stopping existing container...") + old.stop() + old.remove() + except docker.errors.NotFound: + pass + + print(f"🚀 Starting container...") + container = client.containers.run( + image, name=name, ports={f"{port}/tcp": port}, + detach=True, shm_size="1g", mem_limit="4g", + ) + + print(f"⏳ Waiting for health...") + for _ in range(30): + time.sleep(1) + container.reload() + if container.status == "running": + try: + import requests + if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200: + print(f"✅ Container healthy!") + return container + except: + pass + raise TimeoutError("Container failed to start") + +async def main(): + print("="*60) + print("TEST 5: Pool Stress - Mixed Configs") + print("="*60) + + client = docker.from_env() + container = None + monitor_thread = None + + try: + container = start_container(client, IMAGE, CONTAINER_NAME, PORT) + + print(f"\n⏳ Waiting for permanent browser init (3s)...") + await asyncio.sleep(3) + + # Start monitoring + stop_monitoring.clear() + stats_history.clear() + monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True) + monitor_thread.start() + + await asyncio.sleep(1) + baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0 + print(f"📏 Baseline: {baseline_mem:.1f} MB\n") + + url = f"http://localhost:{PORT}/crawl" + + print(f"Testing {len(VIEWPORT_CONFIGS)} different configs:") + for i, vp in enumerate(VIEWPORT_CONFIGS): + vp_str = "Default" if vp is None else f"{vp['width']}x{vp['height']}" + print(f" {i+1}. {vp_str}") + print() + + # Run requests: repeat each config REQUESTS_PER_CONFIG times + all_results = [] + config_sequence = [] + + for _ in range(REQUESTS_PER_CONFIG): + for viewport in VIEWPORT_CONFIGS: + config_sequence.append(viewport) + + # Shuffle to mix configs + random.shuffle(config_sequence) + + print(f"🔄 Running {len(config_sequence)} requests with mixed configs...") + + async with httpx.AsyncClient() as http_client: + for i, viewport in enumerate(config_sequence): + result = await crawl_with_viewport(http_client, url, viewport) + all_results.append(result) + + if (i + 1) % 5 == 0: + vp_str = "default" if result['viewport'] is None else f"{result['viewport']['width']}x{result['viewport']['height']}" + status = "✓" if result.get('success') else "✗" + lat = f"{result.get('latency_ms', 0):.0f}ms" if 'latency_ms' in result else "error" + print(f" [{i+1}/{len(config_sequence)}] {status} {vp_str} - {lat}") + + # Stop monitoring + await asyncio.sleep(2) + stop_monitoring.set() + if monitor_thread: + monitor_thread.join(timeout=2) + + # Analyze results + pool_stats = analyze_pool_logs(container) + + successes = sum(1 for r in all_results if r.get("success")) + success_rate = (successes / len(all_results)) * 100 + latencies = [r["latency_ms"] for r in all_results if "latency_ms" in r] + avg_lat = sum(latencies) / len(latencies) if latencies else 0 + + memory_samples = [s['memory_mb'] for s in stats_history] + peak_mem = max(memory_samples) if memory_samples else 0 + final_mem = memory_samples[-1] if memory_samples else 0 + + print(f"\n{'='*60}") + print(f"RESULTS:") + print(f"{'='*60}") + print(f" Requests: {len(all_results)}") + print(f" Success Rate: {success_rate:.1f}% ({successes}/{len(all_results)})") + print(f" Avg Latency: {avg_lat:.0f}ms") + print(f"\n Pool Statistics:") + print(f" 🔥 Permanent: {pool_stats['permanent']}") + print(f" ♨️ Hot: {pool_stats['hot']}") + print(f" ❄️ Cold: {pool_stats['cold']}") + print(f" 🆕 New: {pool_stats['new']}") + print(f" ⬆️ Promotions: {pool_stats['promotions']}") + print(f" 📊 Reuse: {(pool_stats['total'] / len(all_results) * 100):.1f}%") + print(f"\n Memory:") + print(f" Baseline: {baseline_mem:.1f} MB") + print(f" Peak: {peak_mem:.1f} MB") + print(f" Final: {final_mem:.1f} MB") + print(f" Delta: {final_mem - baseline_mem:+.1f} MB") + print(f"{'='*60}") + + # Pass/Fail + passed = True + + if success_rate < 99: + print(f"❌ FAIL: Success rate {success_rate:.1f}% < 99%") + passed = False + + # Should see promotions since we repeat each config 5 times + if pool_stats['promotions'] < (len(VIEWPORT_CONFIGS) - 1): # -1 for default + print(f"⚠️ WARNING: Only {pool_stats['promotions']} promotions (expected ~{len(VIEWPORT_CONFIGS)-1})") + + # Should have created some browsers for different configs + if pool_stats['new'] == 0: + print(f"⚠️ NOTE: No new browsers created (all used default?)") + + if pool_stats['permanent'] == len(all_results): + print(f"⚠️ NOTE: All requests used permanent browser (configs not varying enough?)") + + if final_mem - baseline_mem > 500: + print(f"⚠️ WARNING: Memory grew {final_mem - baseline_mem:.1f} MB") + + if passed: + print(f"✅ TEST PASSED") + return 0 + else: + return 1 + + except Exception as e: + print(f"\n❌ TEST ERROR: {e}") + import traceback + traceback.print_exc() + return 1 + finally: + stop_monitoring.set() + if container: + print(f"🛑 Stopping container...") + container.stop() + container.remove() + +if __name__ == "__main__": + exit_code = asyncio.run(main()) + exit(exit_code) diff --git a/deploy/docker/tests/test_6_multi_endpoint.py b/deploy/docker/tests/test_6_multi_endpoint.py new file mode 100755 index 00000000..2d532d3b --- /dev/null +++ b/deploy/docker/tests/test_6_multi_endpoint.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +""" +Test 6: Multi-Endpoint Testing +- Tests multiple endpoints together: /html, /screenshot, /pdf, /crawl +- Validates each endpoint works correctly +- Monitors success rates per endpoint +""" +import asyncio +import time +import docker +import httpx +from threading import Thread, Event + +# Config +IMAGE = "crawl4ai-local:latest" +CONTAINER_NAME = "crawl4ai-test" +PORT = 11235 +REQUESTS_PER_ENDPOINT = 10 + +# Stats +stats_history = [] +stop_monitoring = Event() + +def monitor_stats(container): + """Background stats collector.""" + for stat in container.stats(decode=True, stream=True): + if stop_monitoring.is_set(): + break + try: + mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024) + stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage}) + except: + pass + time.sleep(0.5) + +async def test_html(client, base_url, count): + """Test /html endpoint.""" + url = f"{base_url}/html" + results = [] + for _ in range(count): + start = time.time() + try: + resp = await client.post(url, json={"url": "https://httpbin.org/html"}, timeout=30.0) + elapsed = (time.time() - start) * 1000 + results.append({"success": resp.status_code == 200, "latency_ms": elapsed}) + except Exception as e: + results.append({"success": False, "error": str(e)}) + return results + +async def test_screenshot(client, base_url, count): + """Test /screenshot endpoint.""" + url = f"{base_url}/screenshot" + results = [] + for _ in range(count): + start = time.time() + try: + resp = await client.post(url, json={"url": "https://httpbin.org/html"}, timeout=30.0) + elapsed = (time.time() - start) * 1000 + results.append({"success": resp.status_code == 200, "latency_ms": elapsed}) + except Exception as e: + results.append({"success": False, "error": str(e)}) + return results + +async def test_pdf(client, base_url, count): + """Test /pdf endpoint.""" + url = f"{base_url}/pdf" + results = [] + for _ in range(count): + start = time.time() + try: + resp = await client.post(url, json={"url": "https://httpbin.org/html"}, timeout=30.0) + elapsed = (time.time() - start) * 1000 + results.append({"success": resp.status_code == 200, "latency_ms": elapsed}) + except Exception as e: + results.append({"success": False, "error": str(e)}) + return results + +async def test_crawl(client, base_url, count): + """Test /crawl endpoint.""" + url = f"{base_url}/crawl" + results = [] + payload = { + "urls": ["https://httpbin.org/html"], + "browser_config": {}, + "crawler_config": {} + } + for _ in range(count): + start = time.time() + try: + resp = await client.post(url, json=payload, timeout=30.0) + elapsed = (time.time() - start) * 1000 + results.append({"success": resp.status_code == 200, "latency_ms": elapsed}) + except Exception as e: + results.append({"success": False, "error": str(e)}) + return results + +def start_container(client, image, name, port): + """Start container.""" + try: + old = client.containers.get(name) + print(f"🧹 Stopping existing container...") + old.stop() + old.remove() + except docker.errors.NotFound: + pass + + print(f"🚀 Starting container...") + container = client.containers.run( + image, name=name, ports={f"{port}/tcp": port}, + detach=True, shm_size="1g", mem_limit="4g", + ) + + print(f"⏳ Waiting for health...") + for _ in range(30): + time.sleep(1) + container.reload() + if container.status == "running": + try: + import requests + if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200: + print(f"✅ Container healthy!") + return container + except: + pass + raise TimeoutError("Container failed to start") + +async def main(): + print("="*60) + print("TEST 6: Multi-Endpoint Testing") + print("="*60) + + client = docker.from_env() + container = None + monitor_thread = None + + try: + container = start_container(client, IMAGE, CONTAINER_NAME, PORT) + + print(f"\n⏳ Waiting for permanent browser init (3s)...") + await asyncio.sleep(3) + + # Start monitoring + stop_monitoring.clear() + stats_history.clear() + monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True) + monitor_thread.start() + + await asyncio.sleep(1) + baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0 + print(f"📏 Baseline: {baseline_mem:.1f} MB\n") + + base_url = f"http://localhost:{PORT}" + + # Test each endpoint + endpoints = { + "/html": test_html, + "/screenshot": test_screenshot, + "/pdf": test_pdf, + "/crawl": test_crawl, + } + + all_endpoint_stats = {} + + async with httpx.AsyncClient() as http_client: + for endpoint_name, test_func in endpoints.items(): + print(f"🔄 Testing {endpoint_name} ({REQUESTS_PER_ENDPOINT} requests)...") + results = await test_func(http_client, base_url, REQUESTS_PER_ENDPOINT) + + successes = sum(1 for r in results if r.get("success")) + success_rate = (successes / len(results)) * 100 + latencies = [r["latency_ms"] for r in results if "latency_ms" in r] + avg_lat = sum(latencies) / len(latencies) if latencies else 0 + + all_endpoint_stats[endpoint_name] = { + 'success_rate': success_rate, + 'avg_latency': avg_lat, + 'total': len(results), + 'successes': successes + } + + print(f" ✓ Success: {success_rate:.1f}% ({successes}/{len(results)}), Avg: {avg_lat:.0f}ms") + + # Stop monitoring + await asyncio.sleep(1) + stop_monitoring.set() + if monitor_thread: + monitor_thread.join(timeout=2) + + # Final stats + memory_samples = [s['memory_mb'] for s in stats_history] + peak_mem = max(memory_samples) if memory_samples else 0 + final_mem = memory_samples[-1] if memory_samples else 0 + + print(f"\n{'='*60}") + print(f"RESULTS:") + print(f"{'='*60}") + for endpoint, stats in all_endpoint_stats.items(): + print(f" {endpoint:12} Success: {stats['success_rate']:5.1f}% Avg: {stats['avg_latency']:6.0f}ms") + + print(f"\n Memory:") + print(f" Baseline: {baseline_mem:.1f} MB") + print(f" Peak: {peak_mem:.1f} MB") + print(f" Final: {final_mem:.1f} MB") + print(f" Delta: {final_mem - baseline_mem:+.1f} MB") + print(f"{'='*60}") + + # Pass/Fail + passed = True + for endpoint, stats in all_endpoint_stats.items(): + if stats['success_rate'] < 100: + print(f"❌ FAIL: {endpoint} success rate {stats['success_rate']:.1f}% < 100%") + passed = False + + if passed: + print(f"✅ TEST PASSED") + return 0 + else: + return 1 + + except Exception as e: + print(f"\n❌ TEST ERROR: {e}") + import traceback + traceback.print_exc() + return 1 + finally: + stop_monitoring.set() + if container: + print(f"🛑 Stopping container...") + container.stop() + container.remove() + +if __name__ == "__main__": + exit_code = asyncio.run(main()) + exit(exit_code) diff --git a/deploy/docker/tests/test_7_cleanup.py b/deploy/docker/tests/test_7_cleanup.py new file mode 100755 index 00000000..2fdbe9a6 --- /dev/null +++ b/deploy/docker/tests/test_7_cleanup.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +""" +Test 7: Cleanup Verification (Janitor) +- Creates load spike then goes idle +- Verifies memory returns to near baseline +- Tests janitor cleanup of idle browsers +- Monitors memory recovery time +""" +import asyncio +import time +import docker +import httpx +from threading import Thread, Event + +# Config +IMAGE = "crawl4ai-local:latest" +CONTAINER_NAME = "crawl4ai-test" +PORT = 11235 +SPIKE_REQUESTS = 20 # Create some browsers +IDLE_TIME = 90 # Wait 90s for janitor (runs every 60s) + +# Stats +stats_history = [] +stop_monitoring = Event() + +def monitor_stats(container): + """Background stats collector.""" + for stat in container.stats(decode=True, stream=True): + if stop_monitoring.is_set(): + break + try: + mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024) + stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage}) + except: + pass + time.sleep(1) # Sample every 1s for this test + +def start_container(client, image, name, port): + """Start container.""" + try: + old = client.containers.get(name) + print(f"🧹 Stopping existing container...") + old.stop() + old.remove() + except docker.errors.NotFound: + pass + + print(f"🚀 Starting container...") + container = client.containers.run( + image, name=name, ports={f"{port}/tcp": port}, + detach=True, shm_size="1g", mem_limit="4g", + ) + + print(f"⏳ Waiting for health...") + for _ in range(30): + time.sleep(1) + container.reload() + if container.status == "running": + try: + import requests + if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200: + print(f"✅ Container healthy!") + return container + except: + pass + raise TimeoutError("Container failed to start") + +async def main(): + print("="*60) + print("TEST 7: Cleanup Verification (Janitor)") + print("="*60) + + client = docker.from_env() + container = None + monitor_thread = None + + try: + container = start_container(client, IMAGE, CONTAINER_NAME, PORT) + + print(f"\n⏳ Waiting for permanent browser init (3s)...") + await asyncio.sleep(3) + + # Start monitoring + stop_monitoring.clear() + stats_history.clear() + monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True) + monitor_thread.start() + + await asyncio.sleep(2) + baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0 + print(f"📏 Baseline: {baseline_mem:.1f} MB\n") + + # Create load spike with different configs to populate pool + print(f"🔥 Creating load spike ({SPIKE_REQUESTS} requests with varied configs)...") + url = f"http://localhost:{PORT}/crawl" + + viewports = [ + {"width": 1920, "height": 1080}, + {"width": 1024, "height": 768}, + {"width": 375, "height": 667}, + ] + + async with httpx.AsyncClient(timeout=60.0) as http_client: + tasks = [] + for i in range(SPIKE_REQUESTS): + vp = viewports[i % len(viewports)] + payload = { + "urls": ["https://httpbin.org/html"], + "browser_config": { + "type": "BrowserConfig", + "params": { + "viewport": {"type": "dict", "value": vp}, + "headless": True, + "text_mode": True, + "extra_args": [ + "--no-sandbox", "--disable-dev-shm-usage", + "--disable-gpu", "--disable-software-rasterizer", + "--disable-web-security", "--allow-insecure-localhost", + "--ignore-certificate-errors" + ] + } + }, + "crawler_config": {} + } + tasks.append(http_client.post(url, json=payload)) + + results = await asyncio.gather(*tasks, return_exceptions=True) + successes = sum(1 for r in results if hasattr(r, 'status_code') and r.status_code == 200) + print(f" ✓ Spike completed: {successes}/{len(results)} successful") + + # Measure peak + await asyncio.sleep(2) + peak_mem = max([s['memory_mb'] for s in stats_history]) if stats_history else baseline_mem + print(f" 📊 Peak memory: {peak_mem:.1f} MB (+{peak_mem - baseline_mem:.1f} MB)") + + # Now go idle and wait for janitor + print(f"\n⏸️ Going idle for {IDLE_TIME}s (janitor cleanup)...") + print(f" (Janitor runs every 60s, checking for idle browsers)") + + for elapsed in range(0, IDLE_TIME, 10): + await asyncio.sleep(10) + current_mem = stats_history[-1]['memory_mb'] if stats_history else 0 + print(f" [{elapsed+10:3d}s] Memory: {current_mem:.1f} MB") + + # Stop monitoring + stop_monitoring.set() + if monitor_thread: + monitor_thread.join(timeout=2) + + # Analyze memory recovery + final_mem = stats_history[-1]['memory_mb'] if stats_history else 0 + recovery_mb = peak_mem - final_mem + recovery_pct = (recovery_mb / (peak_mem - baseline_mem) * 100) if (peak_mem - baseline_mem) > 0 else 0 + + print(f"\n{'='*60}") + print(f"RESULTS:") + print(f"{'='*60}") + print(f" Memory Journey:") + print(f" Baseline: {baseline_mem:.1f} MB") + print(f" Peak: {peak_mem:.1f} MB (+{peak_mem - baseline_mem:.1f} MB)") + print(f" Final: {final_mem:.1f} MB (+{final_mem - baseline_mem:.1f} MB)") + print(f" Recovered: {recovery_mb:.1f} MB ({recovery_pct:.1f}%)") + print(f"{'='*60}") + + # Pass/Fail + passed = True + + # Should have created some memory pressure + if peak_mem - baseline_mem < 100: + print(f"⚠️ WARNING: Peak increase only {peak_mem - baseline_mem:.1f} MB (expected more browsers)") + + # Should recover most memory (within 100MB of baseline) + if final_mem - baseline_mem > 100: + print(f"⚠️ WARNING: Memory didn't recover well (still +{final_mem - baseline_mem:.1f} MB above baseline)") + else: + print(f"✅ Good memory recovery!") + + # Baseline + 50MB tolerance + if final_mem - baseline_mem < 50: + print(f"✅ Excellent cleanup (within 50MB of baseline)") + + print(f"✅ TEST PASSED") + return 0 + + except Exception as e: + print(f"\n❌ TEST ERROR: {e}") + import traceback + traceback.print_exc() + return 1 + finally: + stop_monitoring.set() + if container: + print(f"🛑 Stopping container...") + container.stop() + container.remove() + +if __name__ == "__main__": + exit_code = asyncio.run(main()) + exit(exit_code) diff --git a/deploy/docker/utils.py b/deploy/docker/utils.py index 5f3618af..52f4e11f 100644 --- a/deploy/docker/utils.py +++ b/deploy/docker/utils.py @@ -178,4 +178,29 @@ def verify_email_domain(email: str) -> bool: records = dns.resolver.resolve(domain, 'MX') return True if records else False except Exception as e: - return False \ No newline at end of file + return False + +def get_container_memory_percent() -> float: + """Get actual container memory usage vs limit (cgroup v1/v2 aware).""" + try: + # Try cgroup v2 first + usage_path = Path("/sys/fs/cgroup/memory.current") + limit_path = Path("/sys/fs/cgroup/memory.max") + if not usage_path.exists(): + # Fall back to cgroup v1 + usage_path = Path("/sys/fs/cgroup/memory/memory.usage_in_bytes") + limit_path = Path("/sys/fs/cgroup/memory/memory.limit_in_bytes") + + usage = int(usage_path.read_text()) + limit = int(limit_path.read_text()) + + # Handle unlimited (v2: "max", v1: > 1e18) + if limit > 1e18: + import psutil + limit = psutil.virtual_memory().total + + return (usage / limit) * 100 + except: + # Non-container or unsupported: fallback to host + import psutil + return psutil.virtual_memory().percent \ No newline at end of file From e2af031b09aab445ae6969010994f5344e8e03a7 Mon Sep 17 00:00:00 2001 From: unclecode Date: Fri, 17 Oct 2025 21:36:25 +0800 Subject: [PATCH 070/119] feat(monitor): add real-time monitoring dashboard with Redis persistence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete observability solution for production deployments with terminal-style UI. **Backend Implementation:** - `monitor.py`: Stats manager tracking requests, browsers, errors, timeline data - `monitor_routes.py`: REST API endpoints for all monitor functionality - GET /monitor/health - System health snapshot - GET /monitor/requests - Active & completed requests - GET /monitor/browsers - Browser pool details - GET /monitor/endpoints/stats - Aggregated endpoint analytics - GET /monitor/timeline - Time-series data (memory, requests, browsers) - GET /monitor/logs/{janitor,errors} - Event logs - POST /monitor/actions/{cleanup,kill_browser,restart_browser} - Control actions - POST /monitor/stats/reset - Reset counters - Redis persistence for endpoint stats (survives restart) - Timeline tracking (5min window, 5s resolution, 60 data points) **Frontend Dashboard** (`/dashboard`): - **System Health Bar**: CPU%, Memory%, Network I/O, Uptime - **Pool Status**: Live counts (permanent/hot/cold browsers + memory) - **Live Activity Tabs**: - Requests: Active (realtime) + recent completed (last 100) - Browsers: Detailed table with actions (kill/restart) - Janitor: Cleanup event log with timestamps - Errors: Recent errors with stack traces - **Endpoint Analytics**: Count, avg latency, success%, pool hit% - **Resource Timeline**: SVG charts (memory/requests/browsers) with terminal aesthetics - **Control Actions**: Force cleanup, restart permanent, reset stats - **Auto-refresh**: 5s polling (toggleable) **Integration:** - Janitor events tracked (close_cold, close_hot, promote) - Crawler pool promotion events logged - Timeline updater background task (5s interval) - Lifespan hooks for monitor initialization **UI Design:** - Terminal vibe matching Crawl4AI theme - Dark background, cyan/pink accents, monospace font - Neon glow effects on charts - Responsive layout, hover interactions - Cross-navigation: Playground ↔ Monitor **Key Features:** - Zero-config: Works out of the box with existing Redis - Real-time visibility into pool efficiency - Manual browser management (kill/restart) - Historical data persistence - DevOps-friendly UX Routes: - API: `/monitor/*` (backend endpoints) - UI: `/dashboard` (static HTML) --- deploy/docker/crawler_pool.py | 28 +- deploy/docker/monitor.py | 305 ++++++++ deploy/docker/monitor_routes.py | 322 ++++++++ deploy/docker/server.py | 42 ++ deploy/docker/static/monitor/index.html | 813 +++++++++++++++++++++ deploy/docker/static/playground/index.html | 13 +- 6 files changed, 1516 insertions(+), 7 deletions(-) create mode 100644 deploy/docker/monitor.py create mode 100644 deploy/docker/monitor_routes.py create mode 100644 deploy/docker/static/monitor/index.html diff --git a/deploy/docker/crawler_pool.py b/deploy/docker/crawler_pool.py index 226e3680..95593b3f 100644 --- a/deploy/docker/crawler_pool.py +++ b/deploy/docker/crawler_pool.py @@ -57,6 +57,14 @@ async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler: if USAGE_COUNT[sig] >= 3: logger.info(f"⬆️ Promoting to hot pool (sig={sig[:8]}, count={USAGE_COUNT[sig]})") HOT_POOL[sig] = COLD_POOL.pop(sig) + + # Track promotion in monitor + try: + from monitor import get_monitor + get_monitor().track_janitor_event("promote", sig, {"count": USAGE_COUNT[sig]}) + except: + pass + return HOT_POOL[sig] logger.info(f"❄️ Using cold pool browser (sig={sig[:8]})") @@ -124,23 +132,39 @@ async def janitor(): # Clean cold pool for sig in list(COLD_POOL.keys()): if now - LAST_USED.get(sig, now) > cold_ttl: - logger.info(f"🧹 Closing cold browser (sig={sig[:8]}, idle={now - LAST_USED[sig]:.0f}s)") + idle_time = now - LAST_USED[sig] + logger.info(f"🧹 Closing cold browser (sig={sig[:8]}, idle={idle_time:.0f}s)") with suppress(Exception): await COLD_POOL[sig].close() COLD_POOL.pop(sig, None) LAST_USED.pop(sig, None) USAGE_COUNT.pop(sig, None) + # Track in monitor + try: + from monitor import get_monitor + get_monitor().track_janitor_event("close_cold", sig, {"idle_seconds": int(idle_time), "ttl": cold_ttl}) + except: + pass + # Clean hot pool (more conservative) for sig in list(HOT_POOL.keys()): if now - LAST_USED.get(sig, now) > hot_ttl: - logger.info(f"🧹 Closing hot browser (sig={sig[:8]}, idle={now - LAST_USED[sig]:.0f}s)") + idle_time = now - LAST_USED[sig] + logger.info(f"🧹 Closing hot browser (sig={sig[:8]}, idle={idle_time:.0f}s)") with suppress(Exception): await HOT_POOL[sig].close() HOT_POOL.pop(sig, None) LAST_USED.pop(sig, None) USAGE_COUNT.pop(sig, None) + # Track in monitor + try: + from monitor import get_monitor + get_monitor().track_janitor_event("close_hot", sig, {"idle_seconds": int(idle_time), "ttl": hot_ttl}) + except: + pass + # Log pool stats if mem_pct > 60: logger.info(f"📊 Pool: hot={len(HOT_POOL)}, cold={len(COLD_POOL)}, mem={mem_pct:.1f}%") diff --git a/deploy/docker/monitor.py b/deploy/docker/monitor.py new file mode 100644 index 00000000..3735280c --- /dev/null +++ b/deploy/docker/monitor.py @@ -0,0 +1,305 @@ +# monitor.py - Real-time monitoring stats with Redis persistence +import time +import json +import asyncio +from typing import Dict, List, Optional +from datetime import datetime, timezone +from collections import deque +from redis import asyncio as aioredis +from utils import get_container_memory_percent +import psutil +import logging + +logger = logging.getLogger(__name__) + +class MonitorStats: + """Tracks real-time server stats with Redis persistence.""" + + def __init__(self, redis: aioredis.Redis): + self.redis = redis + self.start_time = time.time() + + # In-memory queues (fast reads, Redis backup) + self.active_requests: Dict[str, Dict] = {} # id -> request info + self.completed_requests: deque = deque(maxlen=100) # Last 100 + self.janitor_events: deque = deque(maxlen=100) + self.errors: deque = deque(maxlen=100) + + # Endpoint stats (persisted in Redis) + self.endpoint_stats: Dict[str, Dict] = {} # endpoint -> {count, total_time, errors, ...} + + # Timeline data (5min window, 5s resolution = 60 points) + self.memory_timeline: deque = deque(maxlen=60) + self.requests_timeline: deque = deque(maxlen=60) + self.browser_timeline: deque = deque(maxlen=60) + + async def track_request_start(self, request_id: str, endpoint: str, url: str, config: Dict = None): + """Track new request start.""" + req_info = { + "id": request_id, + "endpoint": endpoint, + "url": url[:100], # Truncate long URLs + "start_time": time.time(), + "config_sig": config.get("sig", "default") if config else "default", + "mem_start": psutil.Process().memory_info().rss / (1024 * 1024) + } + self.active_requests[request_id] = req_info + + # Increment endpoint counter + if endpoint not in self.endpoint_stats: + self.endpoint_stats[endpoint] = { + "count": 0, "total_time": 0, "errors": 0, + "pool_hits": 0, "success": 0 + } + self.endpoint_stats[endpoint]["count"] += 1 + + # Persist to Redis (fire and forget) + asyncio.create_task(self._persist_endpoint_stats()) + + async def track_request_end(self, request_id: str, success: bool, error: str = None, + pool_hit: bool = True, status_code: int = 200): + """Track request completion.""" + if request_id not in self.active_requests: + return + + req_info = self.active_requests.pop(request_id) + end_time = time.time() + elapsed = end_time - req_info["start_time"] + mem_end = psutil.Process().memory_info().rss / (1024 * 1024) + mem_delta = mem_end - req_info["mem_start"] + + # Update stats + endpoint = req_info["endpoint"] + if endpoint in self.endpoint_stats: + self.endpoint_stats[endpoint]["total_time"] += elapsed + if success: + self.endpoint_stats[endpoint]["success"] += 1 + else: + self.endpoint_stats[endpoint]["errors"] += 1 + if pool_hit: + self.endpoint_stats[endpoint]["pool_hits"] += 1 + + # Add to completed queue + completed = { + **req_info, + "end_time": end_time, + "elapsed": round(elapsed, 2), + "mem_delta": round(mem_delta, 1), + "success": success, + "error": error, + "status_code": status_code, + "pool_hit": pool_hit + } + self.completed_requests.append(completed) + + # Track errors + if not success and error: + self.errors.append({ + "timestamp": end_time, + "endpoint": endpoint, + "url": req_info["url"], + "error": error, + "request_id": request_id + }) + + await self._persist_endpoint_stats() + + def track_janitor_event(self, event_type: str, sig: str, details: Dict): + """Track janitor cleanup events.""" + self.janitor_events.append({ + "timestamp": time.time(), + "type": event_type, # "close_cold", "close_hot", "promote" + "sig": sig[:8], + "details": details + }) + + async def update_timeline(self): + """Update timeline data points (called every 5s).""" + now = time.time() + mem_pct = get_container_memory_percent() + + # Count requests in last 5s + recent_reqs = sum(1 for req in self.completed_requests + if now - req.get("end_time", 0) < 5) + + # Browser counts (need to import from crawler_pool) + from crawler_pool import PERMANENT, HOT_POOL, COLD_POOL + browser_count = { + "permanent": 1 if PERMANENT else 0, + "hot": len(HOT_POOL), + "cold": len(COLD_POOL) + } + + self.memory_timeline.append({"time": now, "value": mem_pct}) + self.requests_timeline.append({"time": now, "value": recent_reqs}) + self.browser_timeline.append({"time": now, "browsers": browser_count}) + + async def _persist_endpoint_stats(self): + """Persist endpoint stats to Redis.""" + try: + await self.redis.set( + "monitor:endpoint_stats", + json.dumps(self.endpoint_stats), + ex=86400 # 24h TTL + ) + except Exception as e: + logger.warning(f"Failed to persist endpoint stats: {e}") + + async def load_from_redis(self): + """Load persisted stats from Redis.""" + try: + data = await self.redis.get("monitor:endpoint_stats") + if data: + self.endpoint_stats = json.loads(data) + logger.info("Loaded endpoint stats from Redis") + except Exception as e: + logger.warning(f"Failed to load from Redis: {e}") + + def get_health_summary(self) -> Dict: + """Get current system health snapshot.""" + mem_pct = get_container_memory_percent() + cpu_pct = psutil.cpu_percent(interval=0.1) + + # Network I/O (delta since last call) + net = psutil.net_io_counters() + + # Pool status + from crawler_pool import PERMANENT, HOT_POOL, COLD_POOL, LAST_USED + permanent_mem = 270 if PERMANENT else 0 # Estimate + hot_mem = len(HOT_POOL) * 180 # Estimate 180MB per browser + cold_mem = len(COLD_POOL) * 180 + + return { + "container": { + "memory_percent": round(mem_pct, 1), + "cpu_percent": round(cpu_pct, 1), + "network_sent_mb": round(net.bytes_sent / (1024**2), 2), + "network_recv_mb": round(net.bytes_recv / (1024**2), 2), + "uptime_seconds": int(time.time() - self.start_time) + }, + "pool": { + "permanent": {"active": PERMANENT is not None, "memory_mb": permanent_mem}, + "hot": {"count": len(HOT_POOL), "memory_mb": hot_mem}, + "cold": {"count": len(COLD_POOL), "memory_mb": cold_mem}, + "total_memory_mb": permanent_mem + hot_mem + cold_mem + }, + "janitor": { + "next_cleanup_estimate": "adaptive", # Would need janitor state + "memory_pressure": "LOW" if mem_pct < 60 else "MEDIUM" if mem_pct < 80 else "HIGH" + } + } + + def get_active_requests(self) -> List[Dict]: + """Get list of currently active requests.""" + now = time.time() + return [ + { + **req, + "elapsed": round(now - req["start_time"], 1), + "status": "running" + } + for req in self.active_requests.values() + ] + + def get_completed_requests(self, limit: int = 50, filter_status: str = "all") -> List[Dict]: + """Get recent completed requests.""" + requests = list(self.completed_requests)[-limit:] + if filter_status == "success": + requests = [r for r in requests if r.get("success")] + elif filter_status == "error": + requests = [r for r in requests if not r.get("success")] + return requests + + def get_browser_list(self) -> List[Dict]: + """Get detailed browser pool information.""" + from crawler_pool import PERMANENT, HOT_POOL, COLD_POOL, LAST_USED, USAGE_COUNT, DEFAULT_CONFIG_SIG + + browsers = [] + now = time.time() + + if PERMANENT: + browsers.append({ + "type": "permanent", + "sig": DEFAULT_CONFIG_SIG[:8] if DEFAULT_CONFIG_SIG else "unknown", + "age_seconds": int(now - self.start_time), + "last_used_seconds": int(now - LAST_USED.get(DEFAULT_CONFIG_SIG, now)), + "memory_mb": 270, + "hits": USAGE_COUNT.get(DEFAULT_CONFIG_SIG, 0), + "killable": False + }) + + for sig, crawler in HOT_POOL.items(): + browsers.append({ + "type": "hot", + "sig": sig[:8], + "age_seconds": int(now - self.start_time), # Approximation + "last_used_seconds": int(now - LAST_USED.get(sig, now)), + "memory_mb": 180, # Estimate + "hits": USAGE_COUNT.get(sig, 0), + "killable": True + }) + + for sig, crawler in COLD_POOL.items(): + browsers.append({ + "type": "cold", + "sig": sig[:8], + "age_seconds": int(now - self.start_time), + "last_used_seconds": int(now - LAST_USED.get(sig, now)), + "memory_mb": 180, + "hits": USAGE_COUNT.get(sig, 0), + "killable": True + }) + + return browsers + + def get_endpoint_stats_summary(self) -> Dict[str, Dict]: + """Get aggregated endpoint statistics.""" + summary = {} + for endpoint, stats in self.endpoint_stats.items(): + count = stats["count"] + avg_time = (stats["total_time"] / count) if count > 0 else 0 + success_rate = (stats["success"] / count * 100) if count > 0 else 0 + pool_hit_rate = (stats["pool_hits"] / count * 100) if count > 0 else 0 + + summary[endpoint] = { + "count": count, + "avg_latency_ms": round(avg_time * 1000, 1), + "success_rate_percent": round(success_rate, 1), + "pool_hit_rate_percent": round(pool_hit_rate, 1), + "errors": stats["errors"] + } + return summary + + def get_timeline_data(self, metric: str, window: str = "5m") -> Dict: + """Get timeline data for charts.""" + # For now, only 5m window supported + if metric == "memory": + data = list(self.memory_timeline) + elif metric == "requests": + data = list(self.requests_timeline) + elif metric == "browsers": + data = list(self.browser_timeline) + else: + return {"timestamps": [], "values": []} + + return { + "timestamps": [int(d["time"]) for d in data], + "values": [d.get("value", d.get("browsers")) for d in data] + } + + def get_janitor_log(self, limit: int = 100) -> List[Dict]: + """Get recent janitor events.""" + return list(self.janitor_events)[-limit:] + + def get_errors_log(self, limit: int = 100) -> List[Dict]: + """Get recent errors.""" + return list(self.errors)[-limit:] + +# Global instance (initialized in server.py) +monitor_stats: Optional[MonitorStats] = None + +def get_monitor() -> MonitorStats: + """Get global monitor instance.""" + if monitor_stats is None: + raise RuntimeError("Monitor not initialized") + return monitor_stats diff --git a/deploy/docker/monitor_routes.py b/deploy/docker/monitor_routes.py new file mode 100644 index 00000000..e7451468 --- /dev/null +++ b/deploy/docker/monitor_routes.py @@ -0,0 +1,322 @@ +# monitor_routes.py - Monitor API endpoints +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel +from typing import Optional +from monitor import get_monitor +import logging + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/monitor", tags=["monitor"]) + + +@router.get("/health") +async def get_health(): + """Get current system health snapshot.""" + try: + monitor = get_monitor() + return monitor.get_health_summary() + except Exception as e: + logger.error(f"Error getting health: {e}") + raise HTTPException(500, str(e)) + + +@router.get("/requests") +async def get_requests(status: str = "all", limit: int = 50): + """Get active and completed requests. + + Args: + status: Filter by 'active', 'completed', 'success', 'error', or 'all' + limit: Max number of completed requests to return (default 50) + """ + try: + monitor = get_monitor() + + if status == "active": + return {"active": monitor.get_active_requests(), "completed": []} + elif status == "completed": + return {"active": [], "completed": monitor.get_completed_requests(limit)} + elif status in ["success", "error"]: + return {"active": [], "completed": monitor.get_completed_requests(limit, status)} + else: # "all" + return { + "active": monitor.get_active_requests(), + "completed": monitor.get_completed_requests(limit) + } + except Exception as e: + logger.error(f"Error getting requests: {e}") + raise HTTPException(500, str(e)) + + +@router.get("/browsers") +async def get_browsers(): + """Get detailed browser pool information.""" + try: + monitor = get_monitor() + browsers = monitor.get_browser_list() + + # Calculate summary stats + total_browsers = len(browsers) + total_memory = sum(b["memory_mb"] for b in browsers) + + # Calculate reuse rate from recent requests + recent = monitor.get_completed_requests(100) + pool_hits = sum(1 for r in recent if r.get("pool_hit", False)) + reuse_rate = (pool_hits / len(recent) * 100) if recent else 0 + + return { + "browsers": browsers, + "summary": { + "total_count": total_browsers, + "total_memory_mb": total_memory, + "reuse_rate_percent": round(reuse_rate, 1) + } + } + except Exception as e: + logger.error(f"Error getting browsers: {e}") + raise HTTPException(500, str(e)) + + +@router.get("/endpoints/stats") +async def get_endpoint_stats(): + """Get aggregated endpoint statistics.""" + try: + monitor = get_monitor() + return monitor.get_endpoint_stats_summary() + except Exception as e: + logger.error(f"Error getting endpoint stats: {e}") + raise HTTPException(500, str(e)) + + +@router.get("/timeline") +async def get_timeline(metric: str = "memory", window: str = "5m"): + """Get timeline data for charts. + + Args: + metric: 'memory', 'requests', or 'browsers' + window: Time window (only '5m' supported for now) + """ + try: + monitor = get_monitor() + return monitor.get_timeline_data(metric, window) + except Exception as e: + logger.error(f"Error getting timeline: {e}") + raise HTTPException(500, str(e)) + + +@router.get("/logs/janitor") +async def get_janitor_log(limit: int = 100): + """Get recent janitor cleanup events.""" + try: + monitor = get_monitor() + return {"events": monitor.get_janitor_log(limit)} + except Exception as e: + logger.error(f"Error getting janitor log: {e}") + raise HTTPException(500, str(e)) + + +@router.get("/logs/errors") +async def get_errors_log(limit: int = 100): + """Get recent errors.""" + try: + monitor = get_monitor() + return {"errors": monitor.get_errors_log(limit)} + except Exception as e: + logger.error(f"Error getting errors log: {e}") + raise HTTPException(500, str(e)) + + +# ========== Control Actions ========== + +class KillBrowserRequest(BaseModel): + sig: str + + +@router.post("/actions/cleanup") +async def force_cleanup(): + """Force immediate janitor cleanup (kills idle cold pool browsers).""" + try: + from crawler_pool import COLD_POOL, LAST_USED, USAGE_COUNT, LOCK + import time + from contextlib import suppress + + killed_count = 0 + now = time.time() + + async with LOCK: + for sig in list(COLD_POOL.keys()): + # Kill all cold pool browsers immediately + logger.info(f"🧹 Force cleanup: closing cold browser (sig={sig[:8]})") + with suppress(Exception): + await COLD_POOL[sig].close() + COLD_POOL.pop(sig, None) + LAST_USED.pop(sig, None) + USAGE_COUNT.pop(sig, None) + killed_count += 1 + + monitor = get_monitor() + monitor.track_janitor_event("force_cleanup", "manual", {"killed": killed_count}) + + return {"success": True, "killed_browsers": killed_count} + except Exception as e: + logger.error(f"Error during force cleanup: {e}") + raise HTTPException(500, str(e)) + + +@router.post("/actions/kill_browser") +async def kill_browser(req: KillBrowserRequest): + """Kill a specific browser by signature (hot or cold only). + + Args: + sig: Browser config signature (first 8 chars) + """ + try: + from crawler_pool import HOT_POOL, COLD_POOL, LAST_USED, USAGE_COUNT, LOCK, DEFAULT_CONFIG_SIG + from contextlib import suppress + + # Find full signature matching prefix + target_sig = None + pool_type = None + + async with LOCK: + # Check hot pool + for sig in HOT_POOL.keys(): + if sig.startswith(req.sig): + target_sig = sig + pool_type = "hot" + break + + # Check cold pool + if not target_sig: + for sig in COLD_POOL.keys(): + if sig.startswith(req.sig): + target_sig = sig + pool_type = "cold" + break + + # Check if trying to kill permanent + if DEFAULT_CONFIG_SIG and DEFAULT_CONFIG_SIG.startswith(req.sig): + raise HTTPException(403, "Cannot kill permanent browser. Use restart instead.") + + if not target_sig: + raise HTTPException(404, f"Browser with sig={req.sig} not found") + + # Kill the browser + if pool_type == "hot": + browser = HOT_POOL.pop(target_sig) + else: + browser = COLD_POOL.pop(target_sig) + + with suppress(Exception): + await browser.close() + + LAST_USED.pop(target_sig, None) + USAGE_COUNT.pop(target_sig, None) + + logger.info(f"🔪 Killed {pool_type} browser (sig={target_sig[:8]})") + + monitor = get_monitor() + monitor.track_janitor_event("kill_browser", target_sig, {"pool": pool_type, "manual": True}) + + return {"success": True, "killed_sig": target_sig[:8], "pool_type": pool_type} + except HTTPException: + raise + except Exception as e: + logger.error(f"Error killing browser: {e}") + raise HTTPException(500, str(e)) + + +@router.post("/actions/restart_browser") +async def restart_browser(req: KillBrowserRequest): + """Restart a browser (kill + recreate). Works for permanent too. + + Args: + sig: Browser config signature (first 8 chars), or "permanent" + """ + try: + from crawler_pool import (PERMANENT, HOT_POOL, COLD_POOL, LAST_USED, + USAGE_COUNT, LOCK, DEFAULT_CONFIG_SIG, init_permanent) + from crawl4ai import AsyncWebCrawler, BrowserConfig + from contextlib import suppress + import time + + # Handle permanent browser restart + if req.sig == "permanent" or (DEFAULT_CONFIG_SIG and DEFAULT_CONFIG_SIG.startswith(req.sig)): + async with LOCK: + if PERMANENT: + with suppress(Exception): + await PERMANENT.close() + + # Reinitialize permanent + from utils import load_config + config = load_config() + await init_permanent(BrowserConfig( + extra_args=config["crawler"]["browser"].get("extra_args", []), + **config["crawler"]["browser"].get("kwargs", {}), + )) + + logger.info("🔄 Restarted permanent browser") + return {"success": True, "restarted": "permanent"} + + # Handle hot/cold browser restart + target_sig = None + pool_type = None + browser_config = None + + async with LOCK: + # Find browser + for sig in HOT_POOL.keys(): + if sig.startswith(req.sig): + target_sig = sig + pool_type = "hot" + # Would need to reconstruct config (not stored currently) + break + + if not target_sig: + for sig in COLD_POOL.keys(): + if sig.startswith(req.sig): + target_sig = sig + pool_type = "cold" + break + + if not target_sig: + raise HTTPException(404, f"Browser with sig={req.sig} not found") + + # Kill existing + if pool_type == "hot": + browser = HOT_POOL.pop(target_sig) + else: + browser = COLD_POOL.pop(target_sig) + + with suppress(Exception): + await browser.close() + + # Note: We can't easily recreate with same config without storing it + # For now, just kill and let new requests create fresh ones + LAST_USED.pop(target_sig, None) + USAGE_COUNT.pop(target_sig, None) + + logger.info(f"🔄 Restarted {pool_type} browser (sig={target_sig[:8]})") + + monitor = get_monitor() + monitor.track_janitor_event("restart_browser", target_sig, {"pool": pool_type}) + + return {"success": True, "restarted_sig": target_sig[:8], "note": "Browser will be recreated on next request"} + except HTTPException: + raise + except Exception as e: + logger.error(f"Error restarting browser: {e}") + raise HTTPException(500, str(e)) + + +@router.post("/stats/reset") +async def reset_stats(): + """Reset today's endpoint counters.""" + try: + monitor = get_monitor() + monitor.endpoint_stats.clear() + await monitor._persist_endpoint_stats() + + return {"success": True, "message": "Endpoint stats reset"} + except Exception as e: + logger.error(f"Error resetting stats: {e}") + raise HTTPException(500, str(e)) diff --git a/deploy/docker/server.py b/deploy/docker/server.py index 30639852..efb1cecb 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -16,6 +16,7 @@ from fastapi import Request, Depends from fastapi.responses import FileResponse import base64 import re +import logging from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig from api import ( handle_markdown_request, handle_llm_qa, @@ -112,15 +113,40 @@ AsyncWebCrawler.arun = capped_arun @asynccontextmanager async def lifespan(_: FastAPI): from crawler_pool import init_permanent + from monitor import MonitorStats + import monitor as monitor_module + + # Initialize monitor + monitor_module.monitor_stats = MonitorStats(redis) + await monitor_module.monitor_stats.load_from_redis() + + # Initialize browser pool await init_permanent(BrowserConfig( extra_args=config["crawler"]["browser"].get("extra_args", []), **config["crawler"]["browser"].get("kwargs", {}), )) + + # Start background tasks app.state.janitor = asyncio.create_task(janitor()) + app.state.timeline_updater = asyncio.create_task(_timeline_updater()) + yield + + # Cleanup app.state.janitor.cancel() + app.state.timeline_updater.cancel() await close_all() +async def _timeline_updater(): + """Update timeline data every 5 seconds.""" + from monitor import get_monitor + while True: + await asyncio.sleep(5) + try: + await get_monitor().update_timeline() + except Exception as e: + logger.warning(f"Timeline update error: {e}") + # ───────────────────── FastAPI instance ────────────────────── app = FastAPI( title=config["app"]["title"], @@ -138,6 +164,16 @@ app.mount( name="play", ) +# ── static monitor dashboard ──────────────────────────────── +MONITOR_DIR = pathlib.Path(__file__).parent / "static" / "monitor" +if not MONITOR_DIR.exists(): + raise RuntimeError(f"Monitor assets not found at {MONITOR_DIR}") +app.mount( + "/dashboard", + StaticFiles(directory=MONITOR_DIR, html=True), + name="monitor_ui", +) + @app.get("/") async def root(): @@ -221,6 +257,12 @@ def _safe_eval_config(expr: str) -> dict: # ── job router ────────────────────────────────────────────── app.include_router(init_job_router(redis, config, token_dep)) +# ── monitor router ────────────────────────────────────────── +from monitor_routes import router as monitor_router +app.include_router(monitor_router) + +logger = logging.getLogger(__name__) + # ──────────────────────── Endpoints ────────────────────────── @app.post("/token") async def get_token(req: TokenRequest): diff --git a/deploy/docker/static/monitor/index.html b/deploy/docker/static/monitor/index.html new file mode 100644 index 00000000..2beb9467 --- /dev/null +++ b/deploy/docker/static/monitor/index.html @@ -0,0 +1,813 @@ + + + + + + Crawl4AI Monitor + + + + + + + + +
+

+ 📊 Crawl4AI Monitor + + GitHub stars + +

+ +
+ +
+ + +
+ + + Playground +
+
+ + +
+ +
+

System Health

+ +
+ +
+
+ CPU + --% +
+
+
+
+
+ + +
+
+ Memory + --% +
+
+
+
+
+ + +
+
+ Network + -- +
+
⬆0 MB / ⬇0 MB
+
+ + +
+
+ Uptime + -- +
+
Updated: never
+
+
+ + +
+
+
+ 🔥 Permanent: + INACTIVE (0MB) +
+
+ ♨️ Hot: + 0 (0MB) +
+
+ ❄️ Cold: + 0 (0MB) +
+
+
+ Janitor: adaptive | + Memory pressure: LOW +
+
+
+ + +
+
+ + + + +
+ +
+ +
+
+

Active Requests (0)

+ +
+ +
+
+
No active requests
+
+ +

Recent Completed

+
+
No completed requests
+
+
+
+ + + + + + + + + +
+
+ + +
+ +
+

Endpoint Analytics

+
+ + + + + + + + + + + + + +
EndpointCountAvg LatencySuccess%Pool%
No data
+
+
+ + +
+
+

Resource Timeline (5min)

+ +
+ + + + Loading... + +
+
+ + +
+

Control Actions

+
+ + + +
+
+
+
+ + + + diff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.html index 553e6765..510a6620 100644 --- a/deploy/docker/static/playground/index.html +++ b/deploy/docker/static/playground/index.html @@ -167,11 +167,14 @@ -
- - +
+ Monitor +
+ + +
From aba4036ab69b70cc463221ef2a49b07b323ff4eb Mon Sep 17 00:00:00 2001 From: unclecode Date: Fri, 17 Oct 2025 22:43:06 +0800 Subject: [PATCH 071/119] Add demo and test scripts for monitor dashboard activity - Introduced a demo script (`demo_monitor_dashboard.py`) to showcase various monitoring features through simulated activity. - Implemented a test script (`test_monitor_demo.py`) to generate dashboard activity and verify monitor health and endpoint statistics. - Added a logo image to the static assets for branding purposes. --- deploy/docker/api.py | 33 +++- deploy/docker/server.py | 9 + deploy/docker/static/assets/crawl4ai-logo.jpg | Bin 0 -> 5920 bytes deploy/docker/static/assets/crawl4ai-logo.png | Bin 0 -> 1622 bytes deploy/docker/static/assets/logo.png | Bin 0 -> 11243 bytes deploy/docker/static/monitor/index.html | 182 ++++++++---------- deploy/docker/tests/demo_monitor_dashboard.py | 164 ++++++++++++++++ deploy/docker/tests/test_monitor_demo.py | 57 ++++++ 8 files changed, 338 insertions(+), 107 deletions(-) create mode 100644 deploy/docker/static/assets/crawl4ai-logo.jpg create mode 100644 deploy/docker/static/assets/crawl4ai-logo.png create mode 100644 deploy/docker/static/assets/logo.png create mode 100755 deploy/docker/tests/demo_monitor_dashboard.py create mode 100644 deploy/docker/tests/test_monitor_demo.py diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 605b0c8a..64ac4a85 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -460,12 +460,22 @@ async def handle_crawl_request( hooks_config: Optional[dict] = None ) -> dict: """Handle non-streaming crawl requests with optional hooks.""" + # Track request start + request_id = f"req_{uuid4().hex[:8]}" + try: + from monitor import get_monitor + await get_monitor().track_request_start( + request_id, "/crawl", urls[0] if urls else "batch", browser_config + ) + except: + pass # Monitor not critical + start_mem_mb = _get_memory_mb() # <--- Get memory before start_time = time.time() mem_delta_mb = None peak_mem_mb = start_mem_mb hook_manager = None - + try: urls = [('https://' + url) if not url.startswith(('http://', 'https://')) and not url.startswith(("raw:", "raw://")) else url for url in urls] browser_config = BrowserConfig.load(browser_config) @@ -570,7 +580,16 @@ async def handle_crawl_request( "server_memory_delta_mb": mem_delta_mb, "server_peak_memory_mb": peak_mem_mb } - + + # Track request completion + try: + from monitor import get_monitor + await get_monitor().track_request_end( + request_id, success=True, pool_hit=True, status_code=200 + ) + except: + pass + # Add hooks information if hooks were used if hooks_config and hook_manager: from hook_manager import UserHookManager @@ -599,6 +618,16 @@ async def handle_crawl_request( except Exception as e: logger.error(f"Crawl error: {str(e)}", exc_info=True) + + # Track request error + try: + from monitor import get_monitor + await get_monitor().track_request_end( + request_id, success=False, error=str(e), status_code=500 + ) + except: + pass + if 'crawler' in locals() and crawler.ready: # Check if crawler was initialized and started # try: # await crawler.close() diff --git a/deploy/docker/server.py b/deploy/docker/server.py index efb1cecb..364f4457 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -174,6 +174,15 @@ app.mount( name="monitor_ui", ) +# ── static assets (logo, etc) ──────────────────────────────── +ASSETS_DIR = pathlib.Path(__file__).parent / "static" / "assets" +if ASSETS_DIR.exists(): + app.mount( + "/static/assets", + StaticFiles(directory=ASSETS_DIR), + name="assets", + ) + @app.get("/") async def root(): diff --git a/deploy/docker/static/assets/crawl4ai-logo.jpg b/deploy/docker/static/assets/crawl4ai-logo.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6a808c043126f1c691e7bbf81c766a4980b734cf GIT binary patch literal 5920 zcmb7IWmHvL*WM@SI&c8#JakBRN`rJsDi}1dl~q97#=0uqA4 zx4rki_xtmWcZ_eI^<$5*=UijW{XFZLbDrzD>#qPpQ&mG1fIt8M0{(#O1waXaL7{&Z zXmD_0;bLLI;aK=MIM}#^_=JQ6_yhz*#AFB}Vp3uP0t6+36iH4&K|x4zgNl-zij16s z{Ldy37#IV`!o$MCBPSvtBL9Eebq_#_3k(1QFbE3(C56C9A=iBX9e7CCUkmt;VdLOp zL7;FL_?8yT0}A_-%)bW5g2J$IAlIJ&LNE@BfFZz4heG~+|9?Js`S(kee5%JZUwOyg zw_s)M-jQ4por5)<5q~-4+_v~R?U`0dv(i$d2Ifvcyg2?iyTUdM3lJzfEO)!X0kL)7P$B^{6oe8yI20* zo!zCpE-eOOB5Snq{EggWvb_h|CzgZ01Xev;S6M}Tr9u;kHzDA8X}%>)n#frBr=zX` zUZ>$lC+k0|Bp4qWJFZtqPSJ|9%=k(%c4c~Zl+)d+-_acVDi{w2C8~N^Zx8E=d2Mr1 zFse31e+t}A;%ygL=IDPlhOXq#^i;S8G~R#6Tzp#SE%_&^hpeQ55G(lA+|_ z5BRG9ssn&PYGrUsB*RpcW<8<;bog&355mZf0H9DP$e@3rgk!<5Ay9yvLI4Lr%0x!# zh7<%5#RZXuU_r0M+F7@~j2aEq_((-E0R91u zSN=yg%h+nG>n1l;;2ti7$y)3+&(~L~rXBX8gN{;7_h3g`_It2vV9}Jz^KHmQg5$*j zg>Cm6*kPJ3<+SA&$g4XerG2BzJ|z4nu1hyLn;xT`SkEV*`a)DHCT;D-k)QYsKxZyn&MXb~){mex_fQ8E!(ZsGNw_{H$;%Ck@{Zi#yL+m~t z(-Y++E_KEEOW|ze^c>-Hyrv_SL&x)aIb3?RlpAKdYv((g>p_0`JslRw=#GRxAHoH0 z53KKFer_ZMs#!7eCBMCkg9=n(6B?h7R8(xGhQR-;<%BfqCJ4fkF{CUmLwMv22z?5p=fmKe~CX<~0KmY)V_ z4O^k@)_!kOE)Jp>$=pX%I_G^4$3YGWk^&GI6oQ9?^&dHaKw)qIi@ zQGAw%`Ob>cbFb@O{)E00L@Hw#>9uMg;>j-{K5w3HRdze&*eH}ltBzWCu!b1}iVRyZYTpElzz7n~Xx z8(-P0_D0rm)OF}GRdt*sz-OJlMOb2`TFGoC;fTPAmWWx$?BN3*N0Q^kYJo&Id#TwQ zFU()?xmgb}*jD>KsOp&V?DD?`xMv>=-$?0%>`YcC_K6*9yEAGwEF3GT`@Y;+Lgn?a zGTGK-a>Ll+mJFtMr?rEH3c30SAtkHrbRpYr{gsmo-thx-g}?f%Z}X$xGFz=B#{DX0 zR8Hxw+L!vc8F*yt;Vl3>UnI_i& z{>8-IwZe}&!rn2@jyD}y*vuEX3^bgNu67prCF=@Ae9ot<7avI`RYQ*P0n|Nd{*>Q* z)b^_kZ@I~vx&t}=qa3E>LRN@-tDW_e+__ClnuLa^6kO_R9W@%&627_Lc|Mufql`*w z*1$opTmvAQY^0zffPw&u`#<6VssW4?<*LWMW3>kt^!k{DmO&4}urvmp=UN{1wf? z?nl_ccvkoYUTsNPoJX3YSt@)=ftfC54<9u=KU9&GjuY1R5#uGVqrY3jq?z?nVTxT% zD3eHgav@hYJCCPTmjR{xY1@6(~#JU81nT}Zufi*keR zoh{O&yzZkiCHfCjonL)8kh-Qit0PkuF|T-PXXZ0q?j9kgWNM3sk|v={*gy)dFZdqu<0^K7gxQT zRx@T~nO0QRRGR14fEahp1gK^*<_`!}m&d2~J>DbC1Ijqemh z%XFCMTqbQVnok-{H%{Y}88v+xUG`OFq%00%)7_ul9fen12#C$UEkm8Tq&{o(77!gn zkB27|)1M0+7^Xb8((ITY`xF-u)OZumxR~CJ`5JJ+R`S%DQ(<^CsVkhCJi6PzU|NNG za@g>_6ay8` zwHRy%MR*y+Z&VP-7;yT1wlq3u?nvPI9+3S|2lHL5^&TJF?Y#<`r*f15=`CN&+Eh(6 z?NvEGxeN(6dQA|#_C4T`W*{kcb$yX!vmhBP_5lu^<(E?% z4qPQxX2_q1wdrC75*mztX6)f%@dG!pKigt`^FY_GPx9h5Q1xp_=MTcUStA79C|ACV z3%~rmA&n9!^QZe<&dYj3}0AAN|bAaf@)$?-$ZCNX`UmtUj~c;L?i#@QI}jm>5gpq%xm7O7mDr-w}$$oQW~(S|t~rsxml?h#(S z!Qyk2!Iac)&0w3@TbE1>i4{1Ihn5ssla*UNi{|wG^@(t_iN}U6c-4U;y=mRkU#`$w zfHvFLm6eC>HV15v|3uPWbZU6ia<6okkw0^CL^4UuX+w4aOW%zO_#0LNQRPW% zrb9>H9;2(>gud=o_kh|ZyC@YB+E?j1mzFmj9Ry>pf$Dgtdo8Qp1tp;+Ca4>bP4v*x zEBQn5?VtrOK5lf(d*{>=-nI`kuS2LSA-+irKtQ~;L!ifNd z6blN<%D?3iAY~TRSF}M0xc?E6?@V;9Mw<=j^4!{F&XAOiamNQ__RNkwB$;8@sIcNjH3Ufgkch!)9szL2gS_2Q{cjqVfj*}WZfv3fo_A{J?P zG&5CVlCSJ(JjFMM-JgMi>%!-J+wW)Ri)d$C$Yx8paQ5Hs6gaWL#5-`dUcpXg!$bBE z3@|&wk8wVXa>AnuL5)S-;(b6pc7;QtLkHahc{WP*Y|J~Z$@w3r%QXtJRnoFK=PW?R z91~2fGty~Q1|qt%o08*>Uu0HY8$yBo${)$q!GLY)@ecfy<~^a>x5b$z_TBRRIm>@* z?6VupID1clVk?kKRw#tt^^J!n_Iml8)1vAS;tYOVM!8rtI6}E^<^h_T+1qV*x0x0a) zEb(=|xnEx7n{Yg0HJHpck2KpeHEOYK^EMyfJpHkuvwr`})mA{#Z9kw!V=n0z9ZiN{ zllGg)8YTGF4>TtUmd=Y(D!HK7e3xBm|$aPFb5Wz$e7kFkj4;Se2pIQ`!Y zV_7K6VOz`QVA*A1rk|XxJ-26Houwbs7Go2l+xmK(+2pJJJ$^xz7#?!n%QCA-qk5P1 z!u*}~YZH{Yb@kSCP4sukTNt?w1{-SjCesiq;d=c?Jbp~Sh{sDJ-rN!Uv3u&Z{2(pG zGc=Ki2Pdd9qGL!SA!y-4ePI3E$8kKz&yP!;@EN?X0lc5urbf;8wG3kS+zPUh1;5_P zwtW=k7Mly&n}I?K$)G$9>(lRCd&{yv-oCn0<#pS(X~VWkc}+4x8UKDIx?d8lT^34+Es~xKSBkpW=>UC^-9mJr8Bk)D^8d&WDE9_q*CGe**BaX*q z{@dD2J07RL20*8=gZqnI9C4tx-0V@FDs2!Lc9f*WRo(d~{;LCW>NM`pUOZR_T(oz= z2U*c2nWOQJ|F%2m58i2ZQt+mNJuVL9-#bmptf&tN+PE|Qxzd>LbqKwG`&`5|kjNvF zfGSwqxIH0mgq#gR)1(W&dCxt>f_|Pw;&ZLw*&&q{ zQx7!;nB@Z<);hoSEqe~FmQ*ToHuzoiE|fND*Hi5_Gs-d~(;?(P2DwZ^Xnv$_E*K%CqMjY1jg*+3qjF=SrD8Iw*P|}T_f`9kFV-aw^oBAY$PIgLltseP4QQmH4ug65bfLC9+0oLwH#Qw1mQzlJ zLr7ek6|y_RJ^}yP02Nq~N#DjD97_;-&tCs$Ecw&aB2HfyFIURA zu`Z;q2NVcXD?xX7Hi`NJwb`80L`N2N9@tGh)@)kY5{{5MO|!BX!$kC@(=ooyuZ;22 zy1$&9$s4yzuejcau`+W~Y}QPYvm{U>b|0-`#@{b^|MI6Ho4V;G-A1O#1RSTk$FSyx zhWUv19yeW6)==v6)kqj14K|Fsn27kTk)xH0$HhPBC5981qX>}B&}To+UZQWhh;w2WPK;s0jI!8UUt-5CHRHa?8W~)1ZQxfu$v(kB!ys<(Ul?m0DO9 zDw!#TbQPf&?M+f|8Dp&EHEt6_^5E{?M?k@XF!Bs)dP_);dhr|H4?3;3vQbFFJ?Jzn z0Y8nn*oEn4>y-N>k7!RPnOi}GnS=+IQh9&(B<^fWl2BC`Yu#1#fr>iv0|%`h-}+}y zfsH-6m$<-+&ubvk*d7*?ku`H~Aa#I0+2-Dvs@nh{EJ>D4Z*b&*XmqK-Jk-&qBh92V z2Aq}EudoP|xPOUN0X58o%8tdZQ$!2U3zpVc#KRay-&~SbyiO^m=n^4TW-@k(hq%n< z;fUhgbvSB;&@HM6n-nLR=1ei<$-7OP)SF6D*zXKL$Y@DRtb(KxzAB9INg#RygQVeL|VX*Wly(A!iPMY@IRQ&s-K zxU?K#@jZ08esraSTIVKPfy+5LL;<44PQk3x`+F?m8y4*J>{mQ2Ho?m)8|`czc7yEF z4<1-J>Q)n$#T*HFPr#vJH!x7baXTInemK{lp0OSlFK$vX6_jQzI8&T?I4Q;==HaGs zuT+0EVV{?BR00-Xicisf4ItcLfc<(!M0l&TCP@m!uYWS0_3ksFcH<>pqCX(9$COeY zt5cFS_@IptkP$q3;o93}UpXMbb9iqaV6(YAZ=*I|#T^T0a*2^5A0gp)ycNp7g*%zd zb%K0{Ow{R#p<^!7#p+{6W%o2_rpI%78a|RvbU}on0}`z1aBTL9&cyWLWy%l*mpdST zQWu-aT1v$FYk>&x9J2@IQ_ipt5ve2@C9tYhov4@#-1jER+$K$0Y@@03$aq7oo3#>adFU zUmQ?AIn^`MyNp9^8p=ybONOPzYrgG9g|Fq0*(C@QBDqOr+y@%vtcwnE)O}twfAX^? zdhV?4ETEbN5_R*&y4S(Rn|G0O>+a}y~JU12>}FcPzs4AiaugUcoPT&g$FMBfPuuQTw96=z1kLO0f{Z; zl3wJryPprUr)STeJ*7;t+1=TH{_mS_{`u!$p|vJ5O(3|@^NnNJJ#Ab7h8z>rPb}k* z`R0L56nM@d@HEgMhvQXX94(i?xEzmTTF=nJh5dLtgR~z@j-WtcpsCF7%Gf(uKq)=o z{(Av%nITNqS8xj^AS4ip0*(Waz`&sC%bB%5yglGTm@x!kJ3u&M`Uv=C@DXqa52xX5 z-wyldv#?K`0@ths`ul-U2xD&pVQl*Q!FS$;ec~kSGiSk?8X%o^mCSA7+*i7s1mxbm zaOTbfQzg;eBfF2gtm0< zbc6&VVW58im^cy6p~K({C4gf?ZrciZ^eC820wW{7skq=PSHk)F8$e2kjt+!YzXkU6 z0P#35JnY7UM@4OEj$Uz#IwYw9Y9XjYA?lJ67M;65)peVy^CtE3<+K3mWRlvl%(!Hd z7Qonw6;z!!sk*vZbp8T$NlDfpKrJP8JnrI|u~`L#1fDzr=g$YKs(|iph*Kv`FqM_T zK6nV2Fab!X5&H0b;Ph!AnKVNmJpz|5h5hYbAQAz3{zYi*8lbruNF=~F-UOb14mfuX z;{17_NSOf7Q6L_tjz+0x&ZN5DNHs9T*v?(F!Xa7$)RmP~*RE6D>ZWRKqpq%|28
{Q2l+2>e`>wsg&uDgjw{}ZmNMHs>WvO@^XVQs)A;gfY#K-#YU>qQmVRos=kMe zZU2;(l*ZwlIaH0!hD38Kb!8=Wc{$aMCaPOq24wDBV=E0qf zyX>AA$gF3=7|xnyzE@7Tu*iBvUzT6r(V2x)GoGl8>78_ zRLw2abLJYK$wUsgOTc122@_O$;XUtw}b+sF`2_KHR!ugrZ(^Qd3OaS0`vfSgxZVp@{nCBZ!)Kyj4CD`&2Rc{|vODpxfd07>7 z-V%5^62-=_qQZdm-e>HS8d^fol7c#wqH6fVta(ctbyXGh?AcT;Z6;4#y+&P8;o3{a zcGNO@{{dCQRq7ct#tK41e|_(2~}&G z*&X#)Xic9^OA1=y5M!TzLDfHKlt`thqY;BuC=p@dLa_WLh`V(I~{7yO3*F1MM9UmIY_oGH}u)h`Kt6ix>Tg z$g>OL7PS;6!DLeL)Tvaz{Z7?=o2tE&vDcOrOvU>36;vHJsk&~n=$Buqr%a&^hpB}( z6jhG3jBzva*4bN5KEL;g)2|E$gCDQ`#|K0hfT^k;^HQ z#iT3%9z?)XlsXyL{)#yJBC~_McjTr}$6JDVfr6ehsoO(1FCBCADo~b>=Z7PxLJs=w zmp#{B7W`|>QDrPbegFpa6}Fd!K4c3=|N8oEgT5Df_uG#WC^W-o`4xT|nc$`RAG*CS UCSNaJPyhe`07*qoM6N<$f)VKA_5c6? literal 0 HcmV?d00001 diff --git a/deploy/docker/static/assets/logo.png b/deploy/docker/static/assets/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..259118536616e6ac142775c6b9b9d02cd546eb9d GIT binary patch literal 11243 zcmX|nbwJbK_cthwNHe;-LAs2MP)q z%ikBODwz2Q`6sFeSVbP?^BDCu@&}rooTeNKN=+i}ofSF?3RjS-qMRNSbw5|jn|NsQ z0?s@ZY}A9}eBJj3r*ehojYtYE7o_IN5e@-YkU8ob^dH1WaXX|dd6OpaLM}2reOZXRtf@NLjEDK{MBb*-2@aGxUs)wLe z5tp4fTov_w(a3J~QF9xNnl61$* z_&el`{!LgNW6LJvE+2MpNx4`&M-i07zk}c(2-0i(X$(075`80?;wBB#x=Zn3$Xkrh0&qwE6Ax6~*lxz6-AxOL)$d(1x_rq-xK42S7 zA-V6S@oU^qh&`9(yn4*u5jbACCN}QhsyD#sKaIH>e;OqgNkGng-&Hwfy^)DDjR*zBzk-k8ln9rh0#Pi)acKqv|p=4%IzM({0FUhv^ z*R$u0G=G5~X-2{%B)jxfi543skN}Um)g$)+p?C6s8&N)zB3q|m#3SvYQ&Z_^9aCvN z>Q2Mh|JsbZ^h#ftzBbhV$)MC5J!uE3gbzLEp*Ao78rQ-HJqe%thYIo*0PJW`f%rj{ zooCP4Oon@|?>M?vk>-VhVwsfD++i9C*rLN+btXa}R5g z0L0q<$#IM-mCyZ^KPm%;y(USl+D&Tqgnr8CgX^F7SH)>y{f7x-FXMa3Pg|=CMDA*? zCGF^@Kzom*0vF>V4PV1N^FPoRfdbs~h8)X|55M2eJy|FWqj3SpU{ag^Ym|_^i;}4S zgLXQol$}2nR7PXIa9IdMq?GZfGf@g2WDPIuX8N^zS8E*zR|iJCzno|{zo`*$k_mO4 z7b=^xHRrjQQR%c`HCy1*JQ|P)i~n4w56JHUOl*IPpYN!HrRdOicfMr%Le|$jpi#Q1 z7rUu~Rb9H?8i)QTg!f(k`drsRLRr_AfJTD;xEMS&30@GY!tv^l?y@@=8RuZNy>8)cp+>orZZb?`>`q~u<-Z{A zVEBx67PqIm7(0a}qyN-!OqjW-FYtP{@qX=GQW0n zWf(TM>UQVf(I)A3GOT|iD^`HI-6zGWLy6<0d+;CBHXEm47$aci*c56^rSxmlOLG&aa8hy^1+y z(x@fjH!ff}`36jV*=+s!0VQAE9xmp<;#0bQ4S<7%)-=Q;*675mxn-x`vF23E2Q%Xwe zdms$%H}wc{y^+S0P|Xv!QU}fEH7)0UaitSq9h2E2!6y5GS!-m>GNc=WriHd(M-o5~ zUNy_#uyF+5oXyX*J;3muxp%p!=iZI&G=2^w3Y=9a#ohV{vzw`Vmoz}=xVC=v@2JEV26j;azLzTH7ObI&i}CdDM${tVvzt0%cb&PjDT#e-SN$; zNSAo1lgRvD%WGB8_#W>1xEAKgvtTdu-K7b?apdhAwv786m7$MNsE3MCT_{#~El0?;Zr z(DAUE`pte)yn;QVI$%m)8Pn#FtsDj9*B07-tI)b>w8+$y#<^R73rdOpa0}9f<}_4`)1{6H#NJ|`fXPWWl1-h5 zy{>GhU(=uSk)#(i3Cz8~QYF=y%YXcH?w=0(lHWNIlK9~A~Zf=h;KfEAyFGv%PTEDBl{|&5fg-^sgoinOY z?)P@pKwrlr3mS4<#ciP*c{HB_X0{e@W#rdY0_MBmIW}S}GxXZK&pyFD>pkqqb z9BAYk*{#(5v6AXu#%fo1iH(8q)~;jdM{!Lc#QRrq&F<#1<>$oUPxUs>hU5+e4Y-`u zEvF#geEgI_R;zcmI97y4T0=oCgEidnVnwlhUrBwVw61Pu<7Ov?p2pc`gRrPE3zJ{c!(>uE6|I|&-7L4vealuFq&xoj$_RH@ zZJaLfp)EO*tt3Iw2kNHNxjx@m@){T_L~}}g9So2ECKG+Wa#Y68K)flZ{4>4jUXZ~o z;ewS(J=41K4Jo>rT2B&=R~ZcpBxpJz8G~H&GMIGR*H_VfS56T@b&{f8t;_sfkXvhk z1_CG&u{ZDS%lpabps&pgW%)>*xNKas4_qBzFEEHv?@Ew)4TKBP_<$Jr%py)cb+ti0 zK~tD^(YFU9DD^FZs|D;+MUAVmP_s&>35VF|FOA3Kc*52rh-x0PWvL(`IGqex<|g+K z4Ag!exOq=?D9p~(56H97T1dngE=M?v!on9JLho{V%?>npro5+X>Z~f)WPm$c^+mOs zJwmwve-L3(fiI+P(y<3n9HDWCL8Ez$?`V(qE(O0*L+tYrS7s;f0G|`(yfwS{&gA|r zesC$;tibd7ezg~wa3nI`HGi8u$Qh$4rj0$4n=b#(tyFNv!MW^*5Et zzrN7{>TW~P?#*~_P73K6+)VPi8BM~DF4k4QDh=Qz|84_qrM;rQ{M3W*To&Xg{S(w& zn0uR_73rFtBCt{V^o2ia*&kZr-Y4Lb3hPYQtLW;fA=N->My+;Kc*R39?xXx^uThe5 zQ7S_71I7y4>8O%g#Vo!p$NJNfn@N*J7}9V>qW?!>T7|nb`HEo_M?_#*-L4iA}7P*!={P!OiwKsjzZTGgUOQ_8&r(7Zu68(!~ z8tZ5mkJnm1D)oMN)#5$dpUkv*aPAyy@?D+j2St6Nx2j+C8N))BCj8`X=PQ4bGu1(U zGS(LxYCR9Q2gIPy;LuBah{BWdwZ=5gc?$*DC?fA;yGMu{7ix87u4UU*K3LJZFhj_` zccTGQGyV+18ch{@0JVC^@DRo{)*}r$>qrI1vZvRbddN3bZ1bTz;S=l}|A3@0h=>|1 zNoBrl1eZ6f(BHEp)ywDFgH2oZLHs$(`vsFKB!E+rl!*8Yb3}Ya5ZUO%^S2<3f)#XU z28wz2l#dz$7vmm}_I$1(cI%prtOZwSC0eBm+q^6ggW?& z%!tg0z9N%twAzuw;wI10)!$fs@OKNNFc<5rQ#|5N2NOyUuo0hiy2qMoL|7#Py;z(o+gSdi55_ zDuXOd8_sBuA*K_Z+YcH>5B|;$k%4L=Op zlA?_AgMP_SBXt?V4mw9l1QdfgrYz>Z>++Qju!F-0&c0wG1C_Rn-wD@)W_ge06nT(w z=?ky)p(99xy~ih!97{1Ll^wJm#@w5&qk`7pT+BZB=#Ro@{=dlHpU}c^2I^V9W%1hc zC4XM2v;DhrLgs7~nP{0-kpGN8X8((+WRQ-p>F=FJFuueD*|B*);D?bq%d4A~2 zqNZLqDE1jtYk@)b7-Az7!nspOBE(3V@K!y@w%e^Q>-YWVIiZUWdSajHEu&dwp@J)a zlq*Ud;7=a+rGW2sh`14%$`gBEbBrasv`s&IhLkC_t*UjgAoshdNrqCI;RlJP-n!P4 zT!r;FKa|HRR$&bynDy~z>GEO+76_L@rCLk8r33-pHrv6!;W}&`^tr{Qt5|?HKJ48T zq@kYp*`}(lDX(Y9c12GtA1$6Y6q)c)P=3lI-eSyjH8tU4L@LRlvQ37FBPH&RC@{dD zZdM?#2NO`ZS-^a+xkrGWtxOXiCZUTv@bwvBkQZlB6nR#fnwl9Nlz620mi&a}1A{%pNQj@bK+4v*EqVts!q ze%(kw#HDZe+5~SB?D0^$5K&QKwDj|E;H0`QD)Q0+-Lbd zyhW9hI#ShFp>m)6-XU3eliGV0*CLCHSCD_@{?CP$+}Beo-yIx{Fo~XAb^ILJ&#;ZM zld7C29*1v@PKI1yE#1wh2ZzVeNnqykG)jNhnm?H9)}w!q>H2AM^J57wK!sY^hRK}$ zfpSk%omc4=vTVTV^{WualU`pL$S&wmRjD$NJ=qnv;F0G1j$LHik^(u<((siTp z)9AfdvzkDRe7fAq)sxFA4KY(<>a{aKG)AQp<*`)CgxI+)DceexAq$8**HY|#*3EnP z3%+T`wIg{#YP%pc4HC4Li0Sm^c+zAI0g+d6`!G!#x#J||TW5?wE&ErKA+r1o-f{i6 zZNnA$$Y?BdnG>v_ho%uOF*V4UoNM{!MM~%lj95csBvzPePNe`U^qkTTXYcm;Y8Z-& znQ+QdgS&-hdflrZaq;&AX&W~sB+6v=dqhLRiErD*vq(NLhS1og)3gZo&6*0Nv9WD| z9paj`T}(~0n5&CM!>1=;#Bt)$k`qXuAXJm6cnlE|I>$o~j_*m9xLlKW-2bizO4i}E zS7+J~rm3|jaf+M{?g#k1T_RR@^c1I@_w`?ldh?ashR96YCpoXNTwm6f%|;Oso`LHX z5v!g%D^-Xhc4;zc3T)#+TBPARD|jIKkdDq~)j#Pqs(o>zY%FwH;xc^LBPd%9hR-G(^Ua;M#V*;pgGD=cDW9_J7fJ zlpzxThtR92>Nh91NCNXyQeFY{V1geD5NV{}S)%`T6nW%fImJ$IyfaKR_Ns(6C%5Qz zZ~WVtk&_M^%gl6h5AQUcr`)VS6h{H>Nn}vUw00bXHC(h$I8u`8XvZG~Dq2C7(s2gl zn%K94Xlyo*DbUMYp~N#;rUD{oE%TJf;?4apM|nK*_<=WT=Z{K`L9c`LQTKhw%(_F4 zy%SJSq@7|tS$AL;IEN%QHZozjT^XqxVZ1%daY^7`+aJLyI==*9yk_?7$e7br@2(cf zUf5a5(4y_5QmIM|XBuYrFiJBsA$tt&h2HM`j8pn-NHA1j!i^&P{AJE_QToYDWNaIP zT?l5GVMPwq*{AwPTA{0j~|{PtB{zYAJ;<6eV&z|p(G`>&IDWr`6=Zcq-gQXOc*s7f5p^z3@O<8_r4|qos_CVP?Zc}+)YRv3@VMXE{vz?G% zb-%PKQhz7w`xq22MLJusu8Jp)S5ayQaw==O91qJ1UA`^=L*!Hjd5k=bUEevca_DJ9U3c|Sy`}2vKOAxzza51*;I>FCKYBq~Le#@BK{ss3%(WY}{ zk<(C^&A#yZl)0co!iL|2I)CwS)uv?lOYbbmy97u}R1W3ZRHOT<>qePHaI-WM?s_#| zm>W1P^3Y{j#OP+->3Ui;85Da^Db?({b`|_HEd2E6-zdXn)-?*Veacy@|C}InZkMkD}-oE)>7$YwS}s8r%{;q`*$D55^){D|WLihdeHg zvOZ+Z`ksOXOv#-w9Vzkn0XxfN$nwvV72XWGi;IyH$xhxFhTE)$j zSNbwlpYPPYqh&e&$e^)^{`%ZK_FJ*4{RDH-3;a|(q}i-Woon!aZm^G|moG%6SrUCk z!&lKlpHN!e`m(r=cB6EF34SpM#Kc6wi3!pg{vOuj@ip{NbdnU`Y5hx8m(Y25tV+c< z+p*ro9PU=j8~JJMbBaq8Zyi_N%@0)T$1g6wJqsYvFiq`@j>W&Ch+j$J&UTUXcixPd zA>5|ZFstwxBC+xKx(*Z&QND>in`;&_gaiG=iQ<~SvX8~GC?dL82T5#rVn*M#O$TY( zD2y=f;BTIm*cyR2;tD9^Lp)038;f+m5XTYkKCVK zV^mMb?oC-gfy$rv*86B18hUqo~&HMDX`8cXDCp;}<c~#QdB|OcB)$cJWW>DeR3k5%9(D$9Cuuj-!10X^;8zQ z?@qYoi0zDqvfxtgZF9nKhn?E)btK9c7WhMu)HyDM8G--jG+qCrt7T;Rzx=W{l8Wi%J8 zH8^p)GR|Pj>XYUM0h>kPZt3@nt#!1Dh+yFWqRz>FD-)$J+SN4X-vuGtq7gmnTMbc! zg#|z#zqN!RIC)W5cPZ%8Dc|NRG z1O2XPl`fmtC&!0*ui7^ZtJP(#j(_#t7cQuaW7jB4Ba(3i#BP>7Iw2)2)xRkj$L7^ zs9In2yL#ZxJXKv$fZ05+e31qS40!?B%6V7~YK z@LZA0cG*i)Nwc2)DUP8mF*h5jlsP4@9rctooy#Q!uH*wA1{=T&G6oKbdP6TpiD+t? z8szaP!!B`c(p9lSnCmr~HC)rm&s)DX8+hjC_1ip*8X?dB=e7e}`0V$lQf)rkpODKu zt@p|Pw3QUdZRZ|qYYz6aFVQLLK=i?FDQ;Y$$5t-mjzcL*4y4E;>^v_<*YzH+{C$&L z*gGCB5dWKGM^K4)#^@xcY95R%|K;I>az%1q$NChHI}F#A*C3lH3PYuPL148TJnHo= zx#AY_}87o3au#9(TA==OlL_>V0Av zt|68Xsx&8mLvTgQkXdSn2vpOxBg8$A=z|~1kV~{7=843!gec=_J6*O8#|RzROz|t4 z)U}3z4Zqw4s~Pvk-}vMj4*{dS9`48(0G+M{K_`%E(y`5+*xN=J%=vaA9~LL@JH z&$-1^@p}5PQun2Qp$`GnjcTpRId{&D(7NWAwM5@xx7)NxJeVyF5u2{ZFVk5$-G!SK zGF{semXEYJv>=*H*M)SlP?=^|YDdafhFl17%oa5mJz>)sZKK*SM9eD5I;)@u;d>B^ zh=3T*GoJ{o>xZvM-G23V%6$3K^DbWSRom|w&80#AJidIF#5=7H71ykp5{IrFT*1X% zVZd_JOg|eL!_^k_GZdFDMFi;qi5bhF4J)~>IrW-wq0;VK4Wdz->GQOtgBc6b>jq!| zY8nte2}@`-03H%*@a0noG=WP+4=y;4XwxTOchiUKdU9v;S4}d@^Ylr;Mk{a*7>G?x zzk8vj{$SLUwK)AjQ>SyTI%3WK8|Iz8re&jIcTK<> zwk`-$p*R(OiP}`Y`q^$OM!n6K#}$4f3Lq^y3el{&ypo{Q zB7gm*)31(ka7m(T?|?R3#BEI?q}n!mtjIk9=o9Ez7~cLkg1wEk8#s^pL^4gyJwqWm zW)2@!r1-3lT*F+jTWG04b6sZ=npFhlpBsy6`d{COTrV{ z^|JK#M|U2Z1;_Sa*|LV}a*fb*Ok|y;Q?~k?dSQh;QV|NIykHD?{6-)dLQTxH*WGSP zkXI)d8{KYS?9WWu5tm3J0s5bi)~I9EsLLx`p-lcLj$yVMr5jj;`ET%u^r%bRz<3DX z*4LbZCcP1^N!;#fR{4k8x|#K4CE^0j$q(I=pLy*u{r^lBPsBVX;aM0^F;#WBKXxCZ z8!~EA4VBrFROo0Zuh@MWvnN)C@`{xtW+=tYyF%UCERQvKCRNzTL5pQDX0h=l2^PCj zH07VohPuYgfM>0(+Oy~6sS-bhpWv!j!FP~(1Bhpx?RB_olPyX=Cnr1EGR`FcTO8Fg zsNfS|>P?OB;3lJ1?ilvDEiaVCg7I!_aPZve8M104iIWp` zEFQiF7U0Ihopd7^HtL(7bBhNKHlLX+!?!nofF$xLX9$5Fty&^Y%c53O5EexJHn#z? zf{1FVP)wdZ^=2@9-Es=J8TJlt>S`02J{c_$gH^KSs8+~lKaBlnb#wfU`bd3eo8&YJ zN{qVgRq4AynZUQX80!zTt@Zu|Uswi1Fs9{xs^|!>X&7s;E4|}!5#0187Kuj}F2Q2& z7%daG5J_Tt77;79ZySm}^2J{7$gGrZWRs6-BP3hAjekhKu!IDEiX>Ni&OL-pfMGx_Z#&8I^XWSmz*;q@g;`i016iCHUV5Pn+dU^?| zQ9v}z3z@@fYiB@&rs7dbz8{#`RE-{t+z{4MV=u{L7b8!vr@nHRkfZ~UnL*Z)9{d}K zAG4Oi$0Gh$SF-dduBeWH+iIz<51G!go#~xPUhbm$EL~XLj z4)Kq|H#YX$={lxq+q|I`-|ZMod05M`fZ~aG<%dU}GW)jZfmrI&4==t(9|(&dhL|2NKP1ZV(|*xL970snZ!HSRuJigjFSmQe z{b_%TPv7_)alwgb&J`#4=qa83#9cYDK_{G6@I5@b4~KKeJKY06wiyS^MCR*8UG?_{ zw!ZG`bM?e@98Td9=d3@7z;bQ@LJmYj4)Jpp{>MS;97N9GvmK<_z3=GsU~}j#@QyN4 zj7_+D6jyne|7D~sHKhTwfQCl|VRe+<*f|6GBvhD6p7Kz)lhCRD#;)$~$wOaEJwy6N zGs8*qZk<*Q{)9^ln@$66niC*3)&=*D3r~Zw&7;)DJ3m(%9I0g?ODr7j z^A~YeQ1_+@2rF)5H7^X5Ok>`nC&d}eG@*~1lHRcoIFkNKIZvvpCdab@_Yp1_LsPUj z1THhO#B7}%9vr?bc>Zxg^pXcmQ?8`)X8%$&+8-nE;*(1uDY_!UY1>^f_iX|&Ee7C4 zsuF7PVdI`6fb?TEq1V7zln9V%*_&bEV~LA(D$~S4VT`$H42I%;Pb`J`leoMVPju>^ zD-w~nX8susszK5g5UW%nnlZ+i`7peCoS8pMWD%RLz&t+?$}D}oz(^U>6;`un$Wk;) zFY9c^-(x%DA(TxKKj22`(6ZXyyYy4kM^v4ct5G$6nh>1~Lv259bP1KJM&6&laq3c7 zHOu3?RV7QDCmJ;27lAG!tNI`fav_N)$Xc=|EA>*i^{Ia0d~H?W&Lxs3@oP1F&4+uh zvL`lmy}g}-$KBEfStBE(9sg(xrTD>T)larZu=NTaZ~gWu$>gynH-=3z?tsxRA&2lo z!e0f^!c;lsM-A$swRM)dSalFGU<=#RwEM;t2t)T{&l+vIIpd^m$ez` z@287aN?-VNt#KTXl!A#a-?T>hAABm!{=*A;@5+}+6LzgV4PHbKl`?#{Nmj|2-H55l zRn>jcUZN#U7WmB~G~D${q_}gzNx!tVCzg0y^X?-H$xjHvmr9vq@0+eN#wJef=JP^7%kC~Et6`k` ztD4MnN-YHpr`|RY0VN{x9`Iq>JCjtZl39xs%yxgvP#@%#<+Utd86(Zo8{oMLTt-;S5e^SIl-+B{(+|2x_sJWJkx2`WkF z=o+sB0_c z#**2@dWQ6<+x%~;76?Kwv9_~VBzdiTVoN3g@K;C>O?i|A0NA_J`luNNdlLtw0<| v+u2RM7HZi|2(?}8!mW(^D-%U%^YLjwb6c#i7TsT>E{duWNb$40Wyt>l6

- 📊 Crawl4AI Monitor + Crawl4AI + Monitor GitHub stars @@ -90,7 +91,7 @@
@@ -170,85 +171,78 @@ - -
-
- - - - -
- -
- -
-
-

Active Requests (0)

- + +
+ +
+
+

📝 Requests (0 active)

+ +
+
+
+
No active requests
- -
-
-
No active requests
-
- -

Recent Completed

-
-
No completed requests
-
+

Recent Completed

+
+
No completed requests
+
- -
+ +
@@ -313,34 +307,14 @@ // ========== State Management ========== let autoRefresh = true; let refreshInterval; - const REFRESH_RATE = 5000; // 5 seconds + const REFRESH_RATE = 1000; // 1 second - // ========== Tab Switching ========== - document.querySelectorAll('.activity-tab').forEach(btn => { - btn.addEventListener('click', () => { - const tab = btn.dataset.tab; - - // Update tabs - document.querySelectorAll('.activity-tab').forEach(b => { - b.classList.remove('bg-dark', 'text-primary'); - }); - btn.classList.add('bg-dark', 'text-primary'); - - // Update content - document.querySelectorAll('.activity-content').forEach(c => c.classList.add('hidden')); - document.getElementById(`tab-${tab}`).classList.remove('hidden'); - - // Fetch specific data - if (tab === 'browsers') fetchBrowsers(); - if (tab === 'janitor') fetchJanitorLog(); - if (tab === 'errors') fetchErrors(); - }); - }); + // No more tabs - all sections visible at once! // ========== Auto-refresh Toggle ========== document.getElementById('auto-refresh-toggle').addEventListener('click', function() { autoRefresh = !autoRefresh; - this.textContent = autoRefresh ? 'ON ⚡5s' : 'OFF'; + this.textContent = autoRefresh ? 'ON ⚡1s' : 'OFF'; this.classList.toggle('bg-primary'); this.classList.toggle('bg-dark'); this.classList.toggle('text-dark'); @@ -367,6 +341,9 @@ await Promise.all([ fetchHealth(), fetchRequests(), + fetchBrowsers(), + fetchJanitorLog(), + fetchErrors(), fetchEndpointStats(), fetchTimeline() ]); @@ -475,29 +452,24 @@ const tbody = document.getElementById('browsers-table-body'); if (data.browsers.length === 0) { - tbody.innerHTML = 'No browsers'; + tbody.innerHTML = 'No browsers'; } else { tbody.innerHTML = data.browsers.map(b => { const typeIcon = b.type === 'permanent' ? '🔥' : b.type === 'hot' ? '♨️' : '❄️'; const typeColor = b.type === 'permanent' ? 'text-primary' : b.type === 'hot' ? 'text-accent' : 'text-light'; return ` - - ${typeIcon} ${b.type.toUpperCase()} - ${b.sig} - ${formatSeconds(b.age_seconds)} - ${formatSeconds(b.last_used_seconds)} ago - ${b.memory_mb} MB - ${b.hits} - + + ${typeIcon} + ${b.sig} + ${formatSeconds(b.age_seconds)} + ${formatSeconds(b.last_used_seconds)} + ${b.hits} + ${b.killable ? ` - - + ` : ` - + `} diff --git a/deploy/docker/tests/demo_monitor_dashboard.py b/deploy/docker/tests/demo_monitor_dashboard.py new file mode 100755 index 00000000..699988a5 --- /dev/null +++ b/deploy/docker/tests/demo_monitor_dashboard.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +""" +Monitor Dashboard Demo Script +Generates varied activity to showcase all monitoring features for video recording. +""" +import httpx +import asyncio +import time +from datetime import datetime + +BASE_URL = "http://localhost:11235" + +async def demo_dashboard(): + print("🎬 Monitor Dashboard Demo - Starting...\n") + print(f"📊 Dashboard: {BASE_URL}/dashboard") + print("=" * 60) + + async with httpx.AsyncClient(timeout=60.0) as client: + + # Phase 1: Simple requests (permanent browser) + print("\n🔷 Phase 1: Testing permanent browser pool") + print("-" * 60) + for i in range(5): + print(f" {i+1}/5 Request to /crawl (default config)...") + try: + r = await client.post( + f"{BASE_URL}/crawl", + json={"urls": [f"https://httpbin.org/html?req={i}"], "crawler_config": {}} + ) + print(f" ✅ Status: {r.status_code}, Time: {r.elapsed.total_seconds():.2f}s") + except Exception as e: + print(f" ❌ Error: {e}") + await asyncio.sleep(1) # Small delay between requests + + # Phase 2: Create variant browsers (different configs) + print("\n🔶 Phase 2: Testing cold→hot pool promotion") + print("-" * 60) + viewports = [ + {"width": 1920, "height": 1080}, + {"width": 1280, "height": 720}, + {"width": 800, "height": 600} + ] + + for idx, viewport in enumerate(viewports): + print(f" Viewport {viewport['width']}x{viewport['height']}:") + for i in range(4): # 4 requests each to trigger promotion at 3 + try: + r = await client.post( + f"{BASE_URL}/crawl", + json={ + "urls": [f"https://httpbin.org/json?v={idx}&r={i}"], + "browser_config": {"viewport": viewport}, + "crawler_config": {} + } + ) + print(f" {i+1}/4 ✅ {r.status_code} - Should see cold→hot after 3 uses") + except Exception as e: + print(f" {i+1}/4 ❌ {e}") + await asyncio.sleep(0.5) + + # Phase 3: Concurrent burst (stress pool) + print("\n🔷 Phase 3: Concurrent burst (10 parallel)") + print("-" * 60) + tasks = [] + for i in range(10): + tasks.append( + client.post( + f"{BASE_URL}/crawl", + json={"urls": [f"https://httpbin.org/delay/2?burst={i}"], "crawler_config": {}} + ) + ) + + print(" Sending 10 concurrent requests...") + start = time.time() + results = await asyncio.gather(*tasks, return_exceptions=True) + elapsed = time.time() - start + + successes = sum(1 for r in results if not isinstance(r, Exception) and r.status_code == 200) + print(f" ✅ {successes}/10 succeeded in {elapsed:.2f}s") + + # Phase 4: Multi-endpoint coverage + print("\n🔶 Phase 4: Testing multiple endpoints") + print("-" * 60) + endpoints = [ + ("/md", {"url": "https://httpbin.org/html", "f": "fit", "c": "0"}), + ("/screenshot", {"url": "https://httpbin.org/html"}), + ("/pdf", {"url": "https://httpbin.org/html"}), + ] + + for endpoint, payload in endpoints: + print(f" Testing {endpoint}...") + try: + if endpoint == "/md": + r = await client.post(f"{BASE_URL}{endpoint}", json=payload) + else: + r = await client.post(f"{BASE_URL}{endpoint}", json=payload) + print(f" ✅ {r.status_code}") + except Exception as e: + print(f" ❌ {e}") + await asyncio.sleep(1) + + # Phase 5: Intentional error (to populate errors tab) + print("\n🔷 Phase 5: Generating error examples") + print("-" * 60) + print(" Triggering invalid URL error...") + try: + r = await client.post( + f"{BASE_URL}/crawl", + json={"urls": ["invalid://bad-url"], "crawler_config": {}} + ) + print(f" Response: {r.status_code}") + except Exception as e: + print(f" ✅ Error captured: {type(e).__name__}") + + # Phase 6: Wait for janitor activity + print("\n🔶 Phase 6: Waiting for janitor cleanup...") + print("-" * 60) + print(" Idle for 40s to allow janitor to clean cold pool browsers...") + for i in range(40, 0, -10): + print(f" {i}s remaining... (Check dashboard for cleanup events)") + await asyncio.sleep(10) + + # Phase 7: Final stats check + print("\n🔷 Phase 7: Final dashboard state") + print("-" * 60) + + r = await client.get(f"{BASE_URL}/monitor/health") + health = r.json() + print(f" Memory: {health['container']['memory_percent']:.1f}%") + print(f" Browsers: Perm={health['pool']['permanent']['active']}, " + f"Hot={health['pool']['hot']['count']}, Cold={health['pool']['cold']['count']}") + + r = await client.get(f"{BASE_URL}/monitor/endpoints/stats") + stats = r.json() + print(f"\n Endpoint Stats:") + for endpoint, data in stats.items(): + print(f" {endpoint}: {data['count']} req, " + f"{data['avg_latency_ms']:.0f}ms avg, " + f"{data['success_rate_percent']:.1f}% success") + + r = await client.get(f"{BASE_URL}/monitor/browsers") + browsers = r.json() + print(f"\n Pool Efficiency:") + print(f" Total browsers: {browsers['summary']['total_count']}") + print(f" Memory usage: {browsers['summary']['total_memory_mb']} MB") + print(f" Reuse rate: {browsers['summary']['reuse_rate_percent']:.1f}%") + + print("\n" + "=" * 60) + print("✅ Demo complete! Dashboard is now populated with rich data.") + print(f"\n📹 Recording tip: Refresh {BASE_URL}/dashboard") + print(" You should see:") + print(" • Active & completed requests") + print(" • Browser pool (permanent + hot/cold)") + print(" • Janitor cleanup events") + print(" • Endpoint analytics") + print(" • Memory timeline") + +if __name__ == "__main__": + try: + asyncio.run(demo_dashboard()) + except KeyboardInterrupt: + print("\n\n⚠️ Demo interrupted by user") + except Exception as e: + print(f"\n\n❌ Demo failed: {e}") diff --git a/deploy/docker/tests/test_monitor_demo.py b/deploy/docker/tests/test_monitor_demo.py new file mode 100644 index 00000000..2dbff5b1 --- /dev/null +++ b/deploy/docker/tests/test_monitor_demo.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Quick test to generate monitor dashboard activity""" +import httpx +import asyncio + +async def test_dashboard(): + async with httpx.AsyncClient(timeout=30.0) as client: + print("📊 Generating dashboard activity...") + + # Test 1: Simple crawl + print("\n1️⃣ Running simple crawl...") + r1 = await client.post( + "http://localhost:11235/crawl", + json={"urls": ["https://httpbin.org/html"], "crawler_config": {}} + ) + print(f" Status: {r1.status_code}") + + # Test 2: Multiple URLs + print("\n2️⃣ Running multi-URL crawl...") + r2 = await client.post( + "http://localhost:11235/crawl", + json={ + "urls": [ + "https://httpbin.org/html", + "https://httpbin.org/json" + ], + "crawler_config": {} + } + ) + print(f" Status: {r2.status_code}") + + # Test 3: Check monitor health + print("\n3️⃣ Checking monitor health...") + r3 = await client.get("http://localhost:11235/monitor/health") + health = r3.json() + print(f" Memory: {health['container']['memory_percent']}%") + print(f" Browsers: {health['pool']['permanent']['active']}") + + # Test 4: Check requests + print("\n4️⃣ Checking request log...") + r4 = await client.get("http://localhost:11235/monitor/requests") + reqs = r4.json() + print(f" Active: {len(reqs['active'])}") + print(f" Completed: {len(reqs['completed'])}") + + # Test 5: Check endpoint stats + print("\n5️⃣ Checking endpoint stats...") + r5 = await client.get("http://localhost:11235/monitor/endpoints/stats") + stats = r5.json() + for endpoint, data in stats.items(): + print(f" {endpoint}: {data['count']} requests, {data['avg_latency_ms']}ms avg") + + print("\n✅ Dashboard should now show activity!") + print(f"\n🌐 Open: http://localhost:11235/dashboard") + +if __name__ == "__main__": + asyncio.run(test_dashboard()) From 25507adb5bb93e4144157861145db49dc5acd069 Mon Sep 17 00:00:00 2001 From: unclecode Date: Sat, 18 Oct 2025 11:38:25 +0800 Subject: [PATCH 072/119] feat(monitor): implement code review fixes and real-time WebSocket monitoring Backend Improvements (11 fixes applied): Critical Fixes: - Add lock protection for browser pool access in monitor stats - Ensure async track_janitor_event across all call sites - Improve error handling in monitor request tracking (already in place) Important Fixes: - Replace fire-and-forget Redis with background persistence worker - Add time-based expiry for completed requests/errors (5min cleanup) - Implement input validation for monitor route parameters - Add 4s timeout to timeline updater to prevent hangs - Add warning when killing browsers with active requests - Implement monitor cleanup on shutdown with final persistence - Document memory estimates with TODO for actual tracking Frontend Enhancements: WebSocket Real-time Updates: - Add WebSocket endpoint at /monitor/ws for live monitoring - Implement auto-reconnect with exponential backoff (max 5 attempts) - Add graceful fallback to HTTP polling on WebSocket failure - Send comprehensive updates every 2 seconds (health, requests, browsers, timeline, events) UI/UX Improvements: - Add live connection status indicator with pulsing animation - Green "Live" = WebSocket connected - Yellow "Connecting..." = Attempting connection - Blue "Polling" = Fallback to HTTP polling - Red "Disconnected" = Connection failed - Restore original beautiful styling for all sections - Improve request table layout with flex-grow for URL column - Add browser type text labels alongside emojis - Add flex layout to browser section header Testing: - Add test-websocket.py for WebSocket validation - All 7 integration tests passing successfully Summary: 563 additions across 6 files --- deploy/docker/crawler_pool.py | 6 +- deploy/docker/monitor.py | 179 ++++++++++---- deploy/docker/monitor_routes.py | 95 +++++++- deploy/docker/server.py | 13 +- deploy/docker/static/monitor/index.html | 305 +++++++++++++++++++++++- deploy/docker/test-websocket.py | 34 +++ 6 files changed, 561 insertions(+), 71 deletions(-) create mode 100755 deploy/docker/test-websocket.py diff --git a/deploy/docker/crawler_pool.py b/deploy/docker/crawler_pool.py index 95593b3f..509cbba9 100644 --- a/deploy/docker/crawler_pool.py +++ b/deploy/docker/crawler_pool.py @@ -61,7 +61,7 @@ async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler: # Track promotion in monitor try: from monitor import get_monitor - get_monitor().track_janitor_event("promote", sig, {"count": USAGE_COUNT[sig]}) + await get_monitor().track_janitor_event("promote", sig, {"count": USAGE_COUNT[sig]}) except: pass @@ -143,7 +143,7 @@ async def janitor(): # Track in monitor try: from monitor import get_monitor - get_monitor().track_janitor_event("close_cold", sig, {"idle_seconds": int(idle_time), "ttl": cold_ttl}) + await get_monitor().track_janitor_event("close_cold", sig, {"idle_seconds": int(idle_time), "ttl": cold_ttl}) except: pass @@ -161,7 +161,7 @@ async def janitor(): # Track in monitor try: from monitor import get_monitor - get_monitor().track_janitor_event("close_hot", sig, {"idle_seconds": int(idle_time), "ttl": hot_ttl}) + await get_monitor().track_janitor_event("close_hot", sig, {"idle_seconds": int(idle_time), "ttl": hot_ttl}) except: pass diff --git a/deploy/docker/monitor.py b/deploy/docker/monitor.py index 3735280c..469ec36c 100644 --- a/deploy/docker/monitor.py +++ b/deploy/docker/monitor.py @@ -28,6 +28,10 @@ class MonitorStats: # Endpoint stats (persisted in Redis) self.endpoint_stats: Dict[str, Dict] = {} # endpoint -> {count, total_time, errors, ...} + # Background persistence queue (max 10 pending persist requests) + self._persist_queue: asyncio.Queue = asyncio.Queue(maxsize=10) + self._persist_worker_task: Optional[asyncio.Task] = None + # Timeline data (5min window, 5s resolution = 60 points) self.memory_timeline: deque = deque(maxlen=60) self.requests_timeline: deque = deque(maxlen=60) @@ -53,8 +57,11 @@ class MonitorStats: } self.endpoint_stats[endpoint]["count"] += 1 - # Persist to Redis (fire and forget) - asyncio.create_task(self._persist_endpoint_stats()) + # Queue persistence (handled by background worker) + try: + self._persist_queue.put_nowait(True) + except asyncio.QueueFull: + logger.warning("Persistence queue full, skipping") async def track_request_end(self, request_id: str, success: bool, error: str = None, pool_hit: bool = True, status_code: int = 200): @@ -104,7 +111,7 @@ class MonitorStats: await self._persist_endpoint_stats() - def track_janitor_event(self, event_type: str, sig: str, details: Dict): + async def track_janitor_event(self, event_type: str, sig: str, details: Dict): """Track janitor cleanup events.""" self.janitor_events.append({ "timestamp": time.time(), @@ -113,22 +120,43 @@ class MonitorStats: "details": details }) + def _cleanup_old_entries(self, max_age_seconds: int = 300): + """Remove entries older than max_age_seconds (default 5min).""" + now = time.time() + cutoff = now - max_age_seconds + + # Clean completed requests + while self.completed_requests and self.completed_requests[0].get("end_time", 0) < cutoff: + self.completed_requests.popleft() + + # Clean janitor events + while self.janitor_events and self.janitor_events[0].get("timestamp", 0) < cutoff: + self.janitor_events.popleft() + + # Clean errors + while self.errors and self.errors[0].get("timestamp", 0) < cutoff: + self.errors.popleft() + async def update_timeline(self): """Update timeline data points (called every 5s).""" now = time.time() mem_pct = get_container_memory_percent() + # Clean old entries (keep last 5 minutes) + self._cleanup_old_entries(max_age_seconds=300) + # Count requests in last 5s recent_reqs = sum(1 for req in self.completed_requests if now - req.get("end_time", 0) < 5) - # Browser counts (need to import from crawler_pool) - from crawler_pool import PERMANENT, HOT_POOL, COLD_POOL - browser_count = { - "permanent": 1 if PERMANENT else 0, - "hot": len(HOT_POOL), - "cold": len(COLD_POOL) - } + # Browser counts (acquire lock to prevent race conditions) + from crawler_pool import PERMANENT, HOT_POOL, COLD_POOL, LOCK + async with LOCK: + browser_count = { + "permanent": 1 if PERMANENT else 0, + "hot": len(HOT_POOL), + "cold": len(COLD_POOL) + } self.memory_timeline.append({"time": now, "value": mem_pct}) self.requests_timeline.append({"time": now, "value": recent_reqs}) @@ -145,6 +173,47 @@ class MonitorStats: except Exception as e: logger.warning(f"Failed to persist endpoint stats: {e}") + async def _persistence_worker(self): + """Background worker to persist stats to Redis.""" + while True: + try: + await self._persist_queue.get() + await self._persist_endpoint_stats() + self._persist_queue.task_done() + except asyncio.CancelledError: + break + except Exception as e: + logger.error(f"Persistence worker error: {e}") + + def start_persistence_worker(self): + """Start the background persistence worker.""" + if not self._persist_worker_task: + self._persist_worker_task = asyncio.create_task(self._persistence_worker()) + logger.info("Started persistence worker") + + async def stop_persistence_worker(self): + """Stop the background persistence worker.""" + if self._persist_worker_task: + self._persist_worker_task.cancel() + try: + await self._persist_worker_task + except asyncio.CancelledError: + pass + self._persist_worker_task = None + logger.info("Stopped persistence worker") + + async def cleanup(self): + """Cleanup on shutdown - persist final stats and stop workers.""" + logger.info("Monitor cleanup starting...") + try: + # Persist final stats before shutdown + await self._persist_endpoint_stats() + # Stop background worker + await self.stop_persistence_worker() + logger.info("Monitor cleanup completed") + except Exception as e: + logger.error(f"Monitor cleanup error: {e}") + async def load_from_redis(self): """Load persisted stats from Redis.""" try: @@ -155,7 +224,7 @@ class MonitorStats: except Exception as e: logger.warning(f"Failed to load from Redis: {e}") - def get_health_summary(self) -> Dict: + async def get_health_summary(self) -> Dict: """Get current system health snapshot.""" mem_pct = get_container_memory_percent() cpu_pct = psutil.cpu_percent(interval=0.1) @@ -163,11 +232,17 @@ class MonitorStats: # Network I/O (delta since last call) net = psutil.net_io_counters() - # Pool status - from crawler_pool import PERMANENT, HOT_POOL, COLD_POOL, LAST_USED - permanent_mem = 270 if PERMANENT else 0 # Estimate - hot_mem = len(HOT_POOL) * 180 # Estimate 180MB per browser - cold_mem = len(COLD_POOL) * 180 + # Pool status (acquire lock to prevent race conditions) + from crawler_pool import PERMANENT, HOT_POOL, COLD_POOL, LOCK + async with LOCK: + # TODO: Track actual browser process memory instead of estimates + # These are conservative estimates based on typical Chromium usage + permanent_mem = 270 if PERMANENT else 0 # Estimate: ~270MB for permanent browser + hot_mem = len(HOT_POOL) * 180 # Estimate: ~180MB per hot pool browser + cold_mem = len(COLD_POOL) * 180 # Estimate: ~180MB per cold pool browser + permanent_active = PERMANENT is not None + hot_count = len(HOT_POOL) + cold_count = len(COLD_POOL) return { "container": { @@ -178,9 +253,9 @@ class MonitorStats: "uptime_seconds": int(time.time() - self.start_time) }, "pool": { - "permanent": {"active": PERMANENT is not None, "memory_mb": permanent_mem}, - "hot": {"count": len(HOT_POOL), "memory_mb": hot_mem}, - "cold": {"count": len(COLD_POOL), "memory_mb": cold_mem}, + "permanent": {"active": permanent_active, "memory_mb": permanent_mem}, + "hot": {"count": hot_count, "memory_mb": hot_mem}, + "cold": {"count": cold_count, "memory_mb": cold_mem}, "total_memory_mb": permanent_mem + hot_mem + cold_mem }, "janitor": { @@ -210,45 +285,47 @@ class MonitorStats: requests = [r for r in requests if not r.get("success")] return requests - def get_browser_list(self) -> List[Dict]: + async def get_browser_list(self) -> List[Dict]: """Get detailed browser pool information.""" - from crawler_pool import PERMANENT, HOT_POOL, COLD_POOL, LAST_USED, USAGE_COUNT, DEFAULT_CONFIG_SIG + from crawler_pool import PERMANENT, HOT_POOL, COLD_POOL, LAST_USED, USAGE_COUNT, DEFAULT_CONFIG_SIG, LOCK browsers = [] now = time.time() - if PERMANENT: - browsers.append({ - "type": "permanent", - "sig": DEFAULT_CONFIG_SIG[:8] if DEFAULT_CONFIG_SIG else "unknown", - "age_seconds": int(now - self.start_time), - "last_used_seconds": int(now - LAST_USED.get(DEFAULT_CONFIG_SIG, now)), - "memory_mb": 270, - "hits": USAGE_COUNT.get(DEFAULT_CONFIG_SIG, 0), - "killable": False - }) + # Acquire lock to prevent race conditions during iteration + async with LOCK: + if PERMANENT: + browsers.append({ + "type": "permanent", + "sig": DEFAULT_CONFIG_SIG[:8] if DEFAULT_CONFIG_SIG else "unknown", + "age_seconds": int(now - self.start_time), + "last_used_seconds": int(now - LAST_USED.get(DEFAULT_CONFIG_SIG, now)), + "memory_mb": 270, + "hits": USAGE_COUNT.get(DEFAULT_CONFIG_SIG, 0), + "killable": False + }) - for sig, crawler in HOT_POOL.items(): - browsers.append({ - "type": "hot", - "sig": sig[:8], - "age_seconds": int(now - self.start_time), # Approximation - "last_used_seconds": int(now - LAST_USED.get(sig, now)), - "memory_mb": 180, # Estimate - "hits": USAGE_COUNT.get(sig, 0), - "killable": True - }) + for sig, crawler in HOT_POOL.items(): + browsers.append({ + "type": "hot", + "sig": sig[:8], + "age_seconds": int(now - self.start_time), # Approximation + "last_used_seconds": int(now - LAST_USED.get(sig, now)), + "memory_mb": 180, # Estimate + "hits": USAGE_COUNT.get(sig, 0), + "killable": True + }) - for sig, crawler in COLD_POOL.items(): - browsers.append({ - "type": "cold", - "sig": sig[:8], - "age_seconds": int(now - self.start_time), - "last_used_seconds": int(now - LAST_USED.get(sig, now)), - "memory_mb": 180, - "hits": USAGE_COUNT.get(sig, 0), - "killable": True - }) + for sig, crawler in COLD_POOL.items(): + browsers.append({ + "type": "cold", + "sig": sig[:8], + "age_seconds": int(now - self.start_time), + "last_used_seconds": int(now - LAST_USED.get(sig, now)), + "memory_mb": 180, + "hits": USAGE_COUNT.get(sig, 0), + "killable": True + }) return browsers diff --git a/deploy/docker/monitor_routes.py b/deploy/docker/monitor_routes.py index e7451468..fdf156de 100644 --- a/deploy/docker/monitor_routes.py +++ b/deploy/docker/monitor_routes.py @@ -1,9 +1,11 @@ # monitor_routes.py - Monitor API endpoints -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, HTTPException, WebSocket, WebSocketDisconnect from pydantic import BaseModel from typing import Optional from monitor import get_monitor import logging +import asyncio +import json logger = logging.getLogger(__name__) router = APIRouter(prefix="/monitor", tags=["monitor"]) @@ -14,7 +16,7 @@ async def get_health(): """Get current system health snapshot.""" try: monitor = get_monitor() - return monitor.get_health_summary() + return await monitor.get_health_summary() except Exception as e: logger.error(f"Error getting health: {e}") raise HTTPException(500, str(e)) @@ -28,6 +30,12 @@ async def get_requests(status: str = "all", limit: int = 50): status: Filter by 'active', 'completed', 'success', 'error', or 'all' limit: Max number of completed requests to return (default 50) """ + # Input validation + if status not in ["all", "active", "completed", "success", "error"]: + raise HTTPException(400, f"Invalid status: {status}. Must be one of: all, active, completed, success, error") + if limit < 1 or limit > 1000: + raise HTTPException(400, f"Invalid limit: {limit}. Must be between 1 and 1000") + try: monitor = get_monitor() @@ -52,7 +60,7 @@ async def get_browsers(): """Get detailed browser pool information.""" try: monitor = get_monitor() - browsers = monitor.get_browser_list() + browsers = await monitor.get_browser_list() # Calculate summary stats total_browsers = len(browsers) @@ -95,6 +103,12 @@ async def get_timeline(metric: str = "memory", window: str = "5m"): metric: 'memory', 'requests', or 'browsers' window: Time window (only '5m' supported for now) """ + # Input validation + if metric not in ["memory", "requests", "browsers"]: + raise HTTPException(400, f"Invalid metric: {metric}. Must be one of: memory, requests, browsers") + if window != "5m": + raise HTTPException(400, f"Invalid window: {window}. Only '5m' is currently supported") + try: monitor = get_monitor() return monitor.get_timeline_data(metric, window) @@ -106,6 +120,10 @@ async def get_timeline(metric: str = "memory", window: str = "5m"): @router.get("/logs/janitor") async def get_janitor_log(limit: int = 100): """Get recent janitor cleanup events.""" + # Input validation + if limit < 1 or limit > 1000: + raise HTTPException(400, f"Invalid limit: {limit}. Must be between 1 and 1000") + try: monitor = get_monitor() return {"events": monitor.get_janitor_log(limit)} @@ -117,6 +135,10 @@ async def get_janitor_log(limit: int = 100): @router.get("/logs/errors") async def get_errors_log(limit: int = 100): """Get recent errors.""" + # Input validation + if limit < 1 or limit > 1000: + raise HTTPException(400, f"Invalid limit: {limit}. Must be between 1 and 1000") + try: monitor = get_monitor() return {"errors": monitor.get_errors_log(limit)} @@ -154,7 +176,7 @@ async def force_cleanup(): killed_count += 1 monitor = get_monitor() - monitor.track_janitor_event("force_cleanup", "manual", {"killed": killed_count}) + await monitor.track_janitor_event("force_cleanup", "manual", {"killed": killed_count}) return {"success": True, "killed_browsers": killed_count} except Exception as e: @@ -200,6 +222,12 @@ async def kill_browser(req: KillBrowserRequest): if not target_sig: raise HTTPException(404, f"Browser with sig={req.sig} not found") + # Warn if there are active requests (browser might be in use) + monitor = get_monitor() + active_count = len(monitor.get_active_requests()) + if active_count > 0: + logger.warning(f"Killing browser {target_sig[:8]} while {active_count} requests are active - may cause failures") + # Kill the browser if pool_type == "hot": browser = HOT_POOL.pop(target_sig) @@ -215,7 +243,7 @@ async def kill_browser(req: KillBrowserRequest): logger.info(f"🔪 Killed {pool_type} browser (sig={target_sig[:8]})") monitor = get_monitor() - monitor.track_janitor_event("kill_browser", target_sig, {"pool": pool_type, "manual": True}) + await monitor.track_janitor_event("kill_browser", target_sig, {"pool": pool_type, "manual": True}) return {"success": True, "killed_sig": target_sig[:8], "pool_type": pool_type} except HTTPException: @@ -298,7 +326,7 @@ async def restart_browser(req: KillBrowserRequest): logger.info(f"🔄 Restarted {pool_type} browser (sig={target_sig[:8]})") monitor = get_monitor() - monitor.track_janitor_event("restart_browser", target_sig, {"pool": pool_type}) + await monitor.track_janitor_event("restart_browser", target_sig, {"pool": pool_type}) return {"success": True, "restarted_sig": target_sig[:8], "note": "Browser will be recreated on next request"} except HTTPException: @@ -320,3 +348,58 @@ async def reset_stats(): except Exception as e: logger.error(f"Error resetting stats: {e}") raise HTTPException(500, str(e)) + + +@router.websocket("/ws") +async def websocket_endpoint(websocket: WebSocket): + """WebSocket endpoint for real-time monitoring updates. + + Sends updates every 2 seconds with: + - Health stats + - Active/completed requests + - Browser pool status + - Timeline data + """ + await websocket.accept() + logger.info("WebSocket client connected") + + try: + while True: + try: + # Gather all monitoring data + monitor = get_monitor() + + data = { + "timestamp": asyncio.get_event_loop().time(), + "health": await monitor.get_health_summary(), + "requests": { + "active": monitor.get_active_requests(), + "completed": monitor.get_completed_requests(limit=10) + }, + "browsers": await monitor.get_browser_list(), + "timeline": { + "memory": monitor.get_timeline_data("memory", "5m"), + "requests": monitor.get_timeline_data("requests", "5m"), + "browsers": monitor.get_timeline_data("browsers", "5m") + }, + "janitor": monitor.get_janitor_log(limit=10), + "errors": monitor.get_errors_log(limit=10) + } + + # Send update to client + await websocket.send_json(data) + + # Wait 2 seconds before next update + await asyncio.sleep(2) + + except WebSocketDisconnect: + logger.info("WebSocket client disconnected") + break + except Exception as e: + logger.error(f"WebSocket error: {e}", exc_info=True) + await asyncio.sleep(2) # Continue trying + + except Exception as e: + logger.error(f"WebSocket connection error: {e}", exc_info=True) + finally: + logger.info("WebSocket connection closed") diff --git a/deploy/docker/server.py b/deploy/docker/server.py index 364f4457..62e4e441 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -119,6 +119,7 @@ async def lifespan(_: FastAPI): # Initialize monitor monitor_module.monitor_stats = MonitorStats(redis) await monitor_module.monitor_stats.load_from_redis() + monitor_module.monitor_stats.start_persistence_worker() # Initialize browser pool await init_permanent(BrowserConfig( @@ -135,6 +136,14 @@ async def lifespan(_: FastAPI): # Cleanup app.state.janitor.cancel() app.state.timeline_updater.cancel() + + # Monitor cleanup (persist stats and stop workers) + from monitor import get_monitor + try: + await get_monitor().cleanup() + except Exception as e: + logger.error(f"Monitor cleanup failed: {e}") + await close_all() async def _timeline_updater(): @@ -143,7 +152,9 @@ async def _timeline_updater(): while True: await asyncio.sleep(5) try: - await get_monitor().update_timeline() + await asyncio.wait_for(get_monitor().update_timeline(), timeout=4.0) + except asyncio.TimeoutError: + logger.warning("Timeline update timeout after 4s") except Exception as e: logger.warning(f"Timeline update error: {e}") diff --git a/deploy/docker/static/monitor/index.html b/deploy/docker/static/monitor/index.html index f5931fe3..a9f8ed39 100644 --- a/deploy/docker/static/monitor/index.html +++ b/deploy/docker/static/monitor/index.html @@ -35,6 +35,12 @@ } .pulse-slow { animation: pulse-slow 2s ease-in-out infinite; } + @keyframes pulse-fast { + 0%, 100% { opacity: 1; transform: scale(1); } + 50% { opacity: 0.6; transform: scale(1.1); } + } + .pulse-fast { animation: pulse-fast 1s ease-in-out infinite; } + @keyframes spin-slow { from { transform: rotate(0deg); } to { transform: rotate(360deg); } @@ -87,6 +93,14 @@

+ +
+
+
+ Connecting... +
+
+
@@ -196,7 +210,7 @@
-
+

🌐 Browsers (0, 0MB)

Reuse: --%
@@ -308,9 +322,279 @@ let autoRefresh = true; let refreshInterval; const REFRESH_RATE = 1000; // 1 second + let websocket = null; + let wsReconnectAttempts = 0; + const MAX_WS_RECONNECT = 5; + let useWebSocket = true; // Try WebSocket first, fallback to polling // No more tabs - all sections visible at once! + // ========== WebSocket Connection ========== + function updateConnectionStatus(status, message) { + const indicator = document.getElementById('ws-indicator'); + const text = document.getElementById('ws-text'); + + indicator.className = 'w-2 h-2 rounded-full'; + + if (status === 'connected') { + indicator.classList.add('bg-green-500', 'pulse-fast'); + text.textContent = 'Live'; + text.className = 'text-xs text-green-400'; + } else if (status === 'connecting') { + indicator.classList.add('bg-yellow-500', 'pulse-slow'); + text.textContent = 'Connecting...'; + text.className = 'text-xs text-yellow-400'; + } else if (status === 'polling') { + indicator.classList.add('bg-blue-500', 'pulse-slow'); + text.textContent = 'Polling'; + text.className = 'text-xs text-blue-400'; + } else { + indicator.classList.add('bg-red-500'); + text.textContent = message || 'Disconnected'; + text.className = 'text-xs text-red-400'; + } + } + + function connectWebSocket() { + if (wsReconnectAttempts >= MAX_WS_RECONNECT) { + console.log('Max WebSocket reconnect attempts reached, falling back to polling'); + useWebSocket = false; + updateConnectionStatus('polling'); + startAutoRefresh(); + return; + } + + updateConnectionStatus('connecting'); + wsReconnectAttempts++; + + const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; + const wsUrl = `${protocol}//${window.location.host}/monitor/ws`; + + websocket = new WebSocket(wsUrl); + + websocket.onopen = () => { + console.log('WebSocket connected'); + wsReconnectAttempts = 0; + updateConnectionStatus('connected'); + stopAutoRefresh(); // Stop polling if running + }; + + websocket.onmessage = (event) => { + const data = JSON.parse(event.data); + updateDashboard(data); + }; + + websocket.onerror = (error) => { + console.error('WebSocket error:', error); + }; + + websocket.onclose = () => { + console.log('WebSocket closed'); + updateConnectionStatus('disconnected', 'Reconnecting...'); + + if (useWebSocket) { + setTimeout(connectWebSocket, 2000 * wsReconnectAttempts); + } else { + startAutoRefresh(); + } + }; + } + + function updateDashboard(data) { + // Update all dashboard sections with WebSocket data + try { + if (data.health) { + updateHealthDisplay(data.health); + } + if (data.requests) { + updateRequestsDisplay(data.requests); + } + if (data.browsers) { + updateBrowsersDisplay(data.browsers); + } + if (data.janitor) { + updateJanitorDisplay(data.janitor); + } + if (data.errors && data.errors.length > 0) { + updateErrorsDisplay(data.errors); + } + } catch (e) { + console.error('Error updating dashboard:', e); + } + } + + // Helper functions to update displays from WebSocket data + function updateHealthDisplay(health) { + const cpu = health.container.cpu_percent; + const mem = health.container.memory_percent; + + document.getElementById('cpu-percent').textContent = cpu.toFixed(1) + '%'; + document.getElementById('cpu-bar').style.width = Math.min(cpu, 100) + '%'; + document.getElementById('cpu-bar').className = `progress-bar h-2 rounded-full ${cpu > 80 ? 'bg-red-500' : cpu > 60 ? 'bg-yellow-500' : 'bg-primary'}`; + + document.getElementById('mem-percent').textContent = mem.toFixed(1) + '%'; + document.getElementById('mem-bar').style.width = Math.min(mem, 100) + '%'; + document.getElementById('mem-bar').className = `progress-bar h-2 rounded-full ${mem > 80 ? 'bg-red-500' : mem > 60 ? 'bg-yellow-500' : 'bg-accent'}`; + + document.getElementById('net-sent').textContent = health.container.network_sent_mb.toFixed(1); + document.getElementById('net-recv').textContent = health.container.network_recv_mb.toFixed(1); + + const uptime = formatUptime(health.container.uptime_seconds); + document.getElementById('uptime').textContent = uptime; + + const perm = health.pool.permanent; + document.getElementById('pool-perm').textContent = `${perm.active ? 'ACTIVE' : 'INACTIVE'} (${perm.memory_mb}MB)`; + document.getElementById('pool-perm').className = perm.active ? 'text-primary ml-2' : 'text-secondary ml-2'; + + document.getElementById('pool-hot').textContent = `${health.pool.hot.count} (${health.pool.hot.memory_mb}MB)`; + document.getElementById('pool-cold').textContent = `${health.pool.cold.count} (${health.pool.cold.memory_mb}MB)`; + + document.getElementById('janitor-status').textContent = health.janitor.next_cleanup_estimate; + const pressure = health.janitor.memory_pressure; + const pressureEl = document.getElementById('mem-pressure'); + pressureEl.textContent = pressure; + pressureEl.className = pressure === 'HIGH' ? 'text-red-500' : pressure === 'MEDIUM' ? 'text-yellow-500' : 'text-green-500'; + + document.getElementById('last-update').textContent = 'Live: ' + new Date().toLocaleTimeString(); + } + + function updateRequestsDisplay(requests) { + // Update active requests count + const activeCount = document.getElementById('active-count'); + if (activeCount) activeCount.textContent = requests.active.length; + + // Update active requests list + const activeList = document.getElementById('active-requests-list'); + if (activeList) { + if (requests.active.length === 0) { + activeList.innerHTML = '
No active requests
'; + } else { + activeList.innerHTML = requests.active.map(req => ` +
+ ${req.id.substring(0, 8)} + ${req.endpoint} + ${req.url} + ${req.elapsed.toFixed(1)}s + ⏳ +
+ `).join(''); + } + } + + // Update completed requests + const completedList = document.getElementById('completed-requests-list'); + if (completedList) { + if (requests.completed.length === 0) { + completedList.innerHTML = '
No completed requests
'; + } else { + completedList.innerHTML = requests.completed.map(req => ` +
+ ${req.id.substring(0, 8)} + ${req.endpoint} + ${req.url} + ${req.elapsed.toFixed(2)}s + ${req.mem_delta > 0 ? '+' : ''}${req.mem_delta}MB + ${req.success ? '✅' : '❌'} ${req.status_code} +
+ `).join(''); + } + } + } + + function updateBrowsersDisplay(browsers) { + const tbody = document.getElementById('browsers-table-body'); + if (tbody) { + if (browsers.length === 0) { + tbody.innerHTML = 'No browsers'; + } else { + tbody.innerHTML = browsers.map(b => { + const typeIcon = b.type === 'permanent' ? '🔥' : b.type === 'hot' ? '♨️' : '❄️'; + const typeColor = b.type === 'permanent' ? 'text-primary' : b.type === 'hot' ? 'text-accent' : 'text-light'; + + return ` + + ${typeIcon} ${b.type} + ${b.sig} + ${formatSeconds(b.age_seconds || 0)} + ${formatSeconds(b.last_used_seconds || 0)} + ${b.hits} + + ${b.killable ? ` + + ` : ` + + `} + + + `; + }).join(''); + } + } + + // Update browser count and total memory + const countEl = document.getElementById('browser-count'); + if (countEl) countEl.textContent = browsers.length; + + const memEl = document.getElementById('browser-mem'); + if (memEl) { + const totalMem = browsers.reduce((sum, b) => sum + (b.memory_mb || 0), 0); + memEl.textContent = totalMem; + } + + // Update reuse rate (if available from summary data) + // Note: WebSocket sends just browsers array, not summary + // Reuse rate calculation would need to be added to monitor.py + const reuseEl = document.getElementById('reuse-rate'); + if (reuseEl) { + reuseEl.textContent = '---%'; // Not available in real-time yet + } + } + + function updateJanitorDisplay(events) { + const janitorLog = document.getElementById('janitor-log'); + if (janitorLog) { + if (events.length === 0) { + janitorLog.innerHTML = '
No events yet
'; + } else { + janitorLog.innerHTML = events.slice(0, 10).reverse().map(evt => { + const time = new Date(evt.timestamp * 1000).toLocaleTimeString(); + const icon = evt.type === 'close_cold' ? '🧹❄️' : evt.type === 'close_hot' ? '🧹♨️' : '⬆️'; + const details = JSON.stringify(evt.details); + + return `
+ ${time} + ${icon} + ${evt.type} + sig=${evt.sig} + ${details} +
`; + }).join(''); + } + } + } + + function updateErrorsDisplay(errors) { + const errorLog = document.getElementById('errors-log'); + if (errorLog) { + if (errors.length === 0) { + errorLog.innerHTML = '
No errors
'; + } else { + errorLog.innerHTML = errors.slice(0, 10).reverse().map(err => { + const time = new Date(err.timestamp * 1000).toLocaleTimeString(); + + return `
+
+ ${time} + ${err.endpoint} +
+
${err.url}
+
${err.error}
+
`; + }).join(''); + } + } + } + // ========== Auto-refresh Toggle ========== document.getElementById('auto-refresh-toggle').addEventListener('click', function() { autoRefresh = !autoRefresh; @@ -426,13 +710,13 @@ completedList.innerHTML = '
No completed requests
'; } else { completedList.innerHTML = data.completed.map(req => ` -
- ${req.id.substring(0, 8)} - ${req.endpoint} - ${req.url} - ${req.elapsed.toFixed(2)}s - ${req.mem_delta > 0 ? '+' : ''}${req.mem_delta}MB - ${req.success ? '✅' : '❌'} ${req.status_code} +
+ ${req.id.substring(0, 8)} + ${req.endpoint} + ${req.url} + ${req.elapsed.toFixed(2)}s + ${req.mem_delta > 0 ? '+' : ''}${req.mem_delta}MB + ${req.success ? '✅' : '❌'} ${req.status_code}
`).join(''); } @@ -460,7 +744,7 @@ return ` - ${typeIcon} + ${typeIcon} ${b.type} ${b.sig} ${formatSeconds(b.age_seconds)} ${formatSeconds(b.last_used_seconds)} @@ -779,7 +1063,8 @@ document.getElementById('filter-requests')?.addEventListener('change', fetchRequests); // ========== Initialize ========== - startAutoRefresh(); + // Try WebSocket first, fallback to polling on failure + connectWebSocket(); diff --git a/deploy/docker/test-websocket.py b/deploy/docker/test-websocket.py new file mode 100755 index 00000000..db121deb --- /dev/null +++ b/deploy/docker/test-websocket.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +""" +Quick WebSocket test - Connect to monitor WebSocket and print updates +""" +import asyncio +import websockets +import json + +async def test_websocket(): + uri = "ws://localhost:11235/monitor/ws" + print(f"Connecting to {uri}...") + + try: + async with websockets.connect(uri) as websocket: + print("✅ Connected!") + + # Receive and print 5 updates + for i in range(5): + message = await websocket.recv() + data = json.loads(message) + print(f"\n📊 Update #{i+1}:") + print(f" - Health: CPU {data['health']['container']['cpu_percent']}%, Memory {data['health']['container']['memory_percent']}%") + print(f" - Active Requests: {len(data['requests']['active'])}") + print(f" - Browsers: {len(data['browsers'])}") + + except Exception as e: + print(f"❌ Error: {e}") + return 1 + + print("\n✅ WebSocket test passed!") + return 0 + +if __name__ == "__main__": + exit(asyncio.run(test_websocket())) From 05921811b8fdf43772c73aca9779c42b86be100f Mon Sep 17 00:00:00 2001 From: unclecode Date: Sat, 18 Oct 2025 12:05:49 +0800 Subject: [PATCH 073/119] docs: add comprehensive technical architecture documentation Created ARCHITECTURE.md as a complete technical reference for the Crawl4AI Docker server, replacing the stress test pipeline document with production-grade documentation. Contents: - System overview with architecture diagrams - Core components deep-dive (server, API, utils) - Smart browser pool implementation details - Real-time monitoring system architecture - WebSocket implementation and fallback strategy - Memory management and container detection - Production optimizations and code review fixes - Deployment guides (local, Docker, production) - Comprehensive troubleshooting section - Debug tools and performance tuning - Test suite documentation - Architecture decision log (ADRs) Target audience: Developers maintaining or extending the system Goal: Enable rapid onboarding and confident modifications --- deploy/docker/ARCHITECTURE.md | 1149 +++++++++++++++++++++++++++++++++ 1 file changed, 1149 insertions(+) create mode 100644 deploy/docker/ARCHITECTURE.md diff --git a/deploy/docker/ARCHITECTURE.md b/deploy/docker/ARCHITECTURE.md new file mode 100644 index 00000000..eb49cdae --- /dev/null +++ b/deploy/docker/ARCHITECTURE.md @@ -0,0 +1,1149 @@ +# Crawl4AI Docker Server - Technical Architecture + +**Version**: 0.7.4 +**Last Updated**: October 2025 +**Status**: Production-ready with real-time monitoring + +This document provides a comprehensive technical overview of the Crawl4AI Docker server architecture, including the smart browser pool, real-time monitoring system, and all production optimizations. + +--- + +## Table of Contents + +1. [System Overview](#system-overview) +2. [Core Components](#core-components) +3. [Smart Browser Pool](#smart-browser-pool) +4. [Real-time Monitoring System](#real-time-monitoring-system) +5. [API Layer](#api-layer) +6. [Memory Management](#memory-management) +7. [Production Optimizations](#production-optimizations) +8. [Deployment & Operations](#deployment--operations) +9. [Troubleshooting & Debugging](#troubleshooting--debugging) + +--- + +## System Overview + +### Architecture Diagram + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Client Requests │ +└────────────┬────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ FastAPI Server (server.py) │ +│ ├─ REST API Endpoints (/crawl, /html, /md, /llm, etc.) │ +│ ├─ WebSocket Endpoint (/monitor/ws) │ +│ └─ Background Tasks (janitor, timeline_updater) │ +└────┬────────────────────┬────────────────────┬──────────────┘ + │ │ │ + ▼ ▼ ▼ +┌─────────────┐ ┌──────────────────┐ ┌─────────────────┐ +│ Browser │ │ Monitor System │ │ Redis │ +│ Pool │ │ (monitor.py) │ │ (Persistence) │ +│ │ │ │ │ │ +│ PERMANENT ●─┤ │ ├─ Stats │ │ ├─ Endpoint │ +│ HOT_POOL ♨─┤ │ ├─ Requests │ │ │ Stats │ +│ COLD_POOL ❄─┤ │ ├─ Browsers │ │ ├─ Task │ +│ │ │ ├─ Timeline │ │ │ Results │ +│ Janitor 🧹─┤ │ └─ Events/Errors │ │ └─ Cache │ +└─────────────┘ └──────────────────┘ └─────────────────┘ +``` + +### Key Features + +- **10x Memory Efficiency**: Smart 3-tier browser pooling reduces memory from 500-700MB to 50-70MB per concurrent user +- **Real-time Monitoring**: WebSocket-based live dashboard with 2-second update intervals +- **Production-Ready**: Comprehensive error handling, timeouts, cleanup, and graceful shutdown +- **Container-Aware**: Accurate memory detection using cgroup v2/v1 +- **Auto-Recovery**: Graceful WebSocket fallback, lock protection, background workers + +--- + +## Core Components + +### 1. Server Core (`server.py`) + +**Responsibilities:** +- FastAPI application lifecycle management +- Route registration and middleware +- Background task orchestration +- Graceful shutdown handling + +**Key Functions:** + +```python +@asynccontextmanager +async def lifespan(app: FastAPI): + """Application lifecycle manager""" + # Startup + - Initialize Redis connection + - Create monitor stats instance + - Start persistence worker + - Initialize permanent browser + - Start janitor (browser cleanup) + - Start timeline updater (5s interval) + + yield + + # Shutdown + - Cancel background tasks + - Persist final monitor stats + - Stop persistence worker + - Close all browsers +``` + +**Configuration:** +- Loaded from `config.yml` +- Browser settings, memory thresholds, rate limiting +- LLM provider credentials +- Server host/port + +### 2. API Layer (`api.py`) + +**Endpoints:** + +| Endpoint | Method | Purpose | Pool Usage | +|----------|--------|---------|------------| +| `/health` | GET | Health check | None | +| `/crawl` | POST | Full crawl with all features | ✓ Pool | +| `/crawl_stream` | POST | Streaming crawl results | ✓ Pool | +| `/html` | POST | HTML extraction | ✓ Pool | +| `/md` | POST | Markdown generation | ✓ Pool | +| `/screenshot` | POST | Page screenshots | ✓ Pool | +| `/pdf` | POST | PDF generation | ✓ Pool | +| `/llm/{path}` | GET/POST | LLM extraction | ✓ Pool | +| `/crawl/job` | POST | Background job creation | ✓ Pool | + +**Request Flow:** + +```python +@app.post("/crawl") +async def crawl(body: CrawlRequest): + # 1. Track request start + request_id = f"req_{uuid4().hex[:8]}" + await get_monitor().track_request_start(request_id, "/crawl", url, config) + + # 2. Get browser from pool + from crawler_pool import get_crawler + crawler = await get_crawler(browser_config) + + # 3. Execute crawl + result = await crawler.arun(url, config=crawler_config) + + # 4. Track request completion + await get_monitor().track_request_end(request_id, success=True) + + # 5. Return result (browser stays in pool) + return result +``` + +### 3. Utility Layer (`utils.py`) + +**Container Memory Detection:** + +```python +def get_container_memory_percent() -> float: + """Accurate container memory detection""" + try: + # Try cgroup v2 first + current = int(Path("/sys/fs/cgroup/memory.current").read_text().strip()) + max_mem = int(Path("/sys/fs/cgroup/memory.max").read_text().strip()) + return (current / max_mem) * 100 + except: + # Fallback to cgroup v1 + usage = int(Path("/sys/fs/cgroup/memory/memory.usage_in_bytes").read_text()) + limit = int(Path("/sys/fs/cgroup/memory/memory.limit_in_bytes").read_text()) + return (usage / limit) * 100 + except: + # Final fallback to psutil (may be inaccurate in containers) + return psutil.virtual_memory().percent +``` + +**Helper Functions:** +- `get_base_url()`: Request base URL extraction +- `is_task_id()`: Task ID validation +- `should_cleanup_task()`: TTL-based cleanup logic +- `validate_llm_provider()`: LLM configuration validation + +--- + +## Smart Browser Pool + +### Architecture + +The browser pool implements a 3-tier strategy optimized for real-world usage patterns: + +``` +┌──────────────────────────────────────────────────────────┐ +│ PERMANENT Browser (Default Config) │ +│ ● Always alive, never cleaned │ +│ ● Serves 90% of requests │ +│ ● ~270MB memory │ +└──────────────────────────────────────────────────────────┘ + ▲ + │ 90% of requests + │ +┌──────────────────────────────────────────────────────────┐ +│ HOT_POOL (Frequently Used Configs) │ +│ ♨ Configs used 3+ times │ +│ ♨ Longer TTL (2-5 min depending on memory) │ +│ ♨ ~180MB per browser │ +└──────────────────────────────────────────────────────────┘ + ▲ + │ Promotion at 3 uses + │ +┌──────────────────────────────────────────────────────────┐ +│ COLD_POOL (Rarely Used Configs) │ +│ ❄ New/rare browser configs │ +│ ❄ Short TTL (30s-5min depending on memory) │ +│ ❄ ~180MB per browser │ +└──────────────────────────────────────────────────────────┘ +``` + +### Implementation (`crawler_pool.py`) + +**Core Data Structures:** + +```python +PERMANENT: Optional[AsyncWebCrawler] = None # Default browser +HOT_POOL: Dict[str, AsyncWebCrawler] = {} # Frequent configs +COLD_POOL: Dict[str, AsyncWebCrawler] = {} # Rare configs +LAST_USED: Dict[str, float] = {} # Timestamp tracking +USAGE_COUNT: Dict[str, int] = {} # Usage counter +LOCK = asyncio.Lock() # Thread-safe access +``` + +**Browser Acquisition Flow:** + +```python +async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler: + sig = _sig(cfg) # SHA1 hash of config + + async with LOCK: # Prevent race conditions + # 1. Check permanent browser + if _is_default_config(sig): + return PERMANENT + + # 2. Check hot pool + if sig in HOT_POOL: + USAGE_COUNT[sig] += 1 + return HOT_POOL[sig] + + # 3. Check cold pool (with promotion logic) + if sig in COLD_POOL: + USAGE_COUNT[sig] += 1 + if USAGE_COUNT[sig] >= 3: + # Promote to hot pool + HOT_POOL[sig] = COLD_POOL.pop(sig) + await get_monitor().track_janitor_event("promote", sig, {...}) + return HOT_POOL[sig] + return COLD_POOL[sig] + + # 4. Memory check before creating new + if get_container_memory_percent() >= MEM_LIMIT: + raise MemoryError(f"Memory at {mem}%, refusing new browser") + + # 5. Create new browser in cold pool + crawler = AsyncWebCrawler(config=cfg) + await crawler.start() + COLD_POOL[sig] = crawler + return crawler +``` + +**Janitor (Adaptive Cleanup):** + +```python +async def janitor(): + """Memory-adaptive browser cleanup""" + while True: + mem_pct = get_container_memory_percent() + + # Adaptive intervals based on memory pressure + if mem_pct > 80: + interval, cold_ttl, hot_ttl = 10, 30, 120 # Aggressive + elif mem_pct > 60: + interval, cold_ttl, hot_ttl = 30, 60, 300 # Moderate + else: + interval, cold_ttl, hot_ttl = 60, 300, 600 # Relaxed + + await asyncio.sleep(interval) + + async with LOCK: + # Clean cold pool first (less valuable) + for sig in list(COLD_POOL.keys()): + if now - LAST_USED[sig] > cold_ttl: + await COLD_POOL[sig].close() + del COLD_POOL[sig], LAST_USED[sig], USAGE_COUNT[sig] + await track_janitor_event("close_cold", sig, {...}) + + # Clean hot pool (more conservative) + for sig in list(HOT_POOL.keys()): + if now - LAST_USED[sig] > hot_ttl: + await HOT_POOL[sig].close() + del HOT_POOL[sig], LAST_USED[sig], USAGE_COUNT[sig] + await track_janitor_event("close_hot", sig, {...}) +``` + +**Config Signature Generation:** + +```python +def _sig(cfg: BrowserConfig) -> str: + """Generate unique signature for browser config""" + payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",",":")) + return hashlib.sha1(payload.encode()).hexdigest() +``` + +--- + +## Real-time Monitoring System + +### Architecture + +The monitoring system provides real-time insights via WebSocket with automatic fallback to HTTP polling. + +**Components:** + +``` +┌─────────────────────────────────────────────────────────┐ +│ MonitorStats Class (monitor.py) │ +│ ├─ In-memory queues (deques with maxlen) │ +│ ├─ Background persistence worker │ +│ ├─ Timeline tracking (5-min window, 5s resolution) │ +│ └─ Time-based expiry (5min for old entries) │ +└───────────┬─────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ WebSocket Endpoint (/monitor/ws) │ +│ ├─ 2-second update intervals │ +│ ├─ Auto-reconnect with exponential backoff │ +│ ├─ Comprehensive data payload │ +│ └─ Graceful fallback to polling │ +└───────────┬─────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ Dashboard UI (static/monitor/index.html) │ +│ ├─ Connection status indicator │ +│ ├─ Live updates (health, requests, browsers) │ +│ ├─ Timeline charts (memory, requests, browsers) │ +│ └─ Janitor events & error logs │ +└─────────────────────────────────────────────────────────┘ +``` + +### Monitor Stats (`monitor.py`) + +**Data Structures:** + +```python +class MonitorStats: + # In-memory queues + active_requests: Dict[str, Dict] # Currently processing + completed_requests: deque(maxlen=100) # Last 100 completed + janitor_events: deque(maxlen=100) # Cleanup events + errors: deque(maxlen=100) # Error log + + # Endpoint stats (persisted to Redis) + endpoint_stats: Dict[str, Dict] # Aggregated stats + + # Timeline data (5min window, 5s resolution = 60 points) + memory_timeline: deque(maxlen=60) + requests_timeline: deque(maxlen=60) + browser_timeline: deque(maxlen=60) + + # Background persistence + _persist_queue: asyncio.Queue(maxsize=10) + _persist_worker_task: Optional[asyncio.Task] +``` + +**Request Tracking:** + +```python +async def track_request_start(request_id, endpoint, url, config): + """Track new request""" + self.active_requests[request_id] = { + "id": request_id, + "endpoint": endpoint, + "url": url, + "start_time": time.time(), + "mem_start": psutil.Process().memory_info().rss / (1024 * 1024) + } + + # Update endpoint stats + if endpoint not in self.endpoint_stats: + self.endpoint_stats[endpoint] = { + "count": 0, "total_time": 0, "errors": 0, + "pool_hits": 0, "success": 0 + } + self.endpoint_stats[endpoint]["count"] += 1 + + # Queue background persistence + self._persist_queue.put_nowait(True) + +async def track_request_end(request_id, success, error=None, ...): + """Track request completion""" + req_info = self.active_requests.pop(request_id) + elapsed = time.time() - req_info["start_time"] + mem_delta = current_mem - req_info["mem_start"] + + # Add to completed queue + self.completed_requests.append({ + "id": request_id, + "endpoint": req_info["endpoint"], + "url": req_info["url"], + "success": success, + "elapsed": elapsed, + "mem_delta": mem_delta, + "end_time": time.time() + }) + + # Update stats + self.endpoint_stats[endpoint]["success" if success else "errors"] += 1 + await self._persist_endpoint_stats() +``` + +**Background Persistence Worker:** + +```python +async def _persistence_worker(self): + """Background worker for Redis persistence""" + while True: + try: + await self._persist_queue.get() + await self._persist_endpoint_stats() + self._persist_queue.task_done() + except asyncio.CancelledError: + break + except Exception as e: + logger.error(f"Persistence worker error: {e}") + +async def _persist_endpoint_stats(self): + """Persist stats to Redis with error handling""" + try: + await self.redis.set( + "monitor:endpoint_stats", + json.dumps(self.endpoint_stats), + ex=86400 # 24h TTL + ) + except Exception as e: + logger.warning(f"Failed to persist endpoint stats: {e}") +``` + +**Time-based Cleanup:** + +```python +def _cleanup_old_entries(self, max_age_seconds=300): + """Remove entries older than 5 minutes""" + now = time.time() + cutoff = now - max_age_seconds + + # Clean completed requests + while self.completed_requests and \ + self.completed_requests[0].get("end_time", 0) < cutoff: + self.completed_requests.popleft() + + # Clean janitor events + while self.janitor_events and \ + self.janitor_events[0].get("timestamp", 0) < cutoff: + self.janitor_events.popleft() + + # Clean errors + while self.errors and \ + self.errors[0].get("timestamp", 0) < cutoff: + self.errors.popleft() +``` + +### WebSocket Implementation (`monitor_routes.py`) + +**Endpoint:** + +```python +@router.websocket("/ws") +async def websocket_endpoint(websocket: WebSocket): + """Real-time monitoring updates""" + await websocket.accept() + logger.info("WebSocket client connected") + + try: + while True: + try: + monitor = get_monitor() + + # Gather comprehensive monitoring data + data = { + "timestamp": time.time(), + "health": await monitor.get_health_summary(), + "requests": { + "active": monitor.get_active_requests(), + "completed": monitor.get_completed_requests(limit=10) + }, + "browsers": await monitor.get_browser_list(), + "timeline": { + "memory": monitor.get_timeline_data("memory", "5m"), + "requests": monitor.get_timeline_data("requests", "5m"), + "browsers": monitor.get_timeline_data("browsers", "5m") + }, + "janitor": monitor.get_janitor_log(limit=10), + "errors": monitor.get_errors_log(limit=10) + } + + await websocket.send_json(data) + await asyncio.sleep(2) # 2-second update interval + + except WebSocketDisconnect: + logger.info("WebSocket client disconnected") + break + except Exception as e: + logger.error(f"WebSocket error: {e}", exc_info=True) + await asyncio.sleep(2) + except Exception as e: + logger.error(f"WebSocket connection error: {e}", exc_info=True) + finally: + logger.info("WebSocket connection closed") +``` + +**Input Validation:** + +```python +@router.get("/requests") +async def get_requests(status: str = "all", limit: int = 50): + # Input validation + if status not in ["all", "active", "completed", "success", "error"]: + raise HTTPException(400, f"Invalid status: {status}") + if limit < 1 or limit > 1000: + raise HTTPException(400, f"Invalid limit: {limit}") + + monitor = get_monitor() + # ... return data +``` + +### Frontend Dashboard + +**Connection Management:** + +```javascript +// WebSocket with auto-reconnect +function connectWebSocket() { + if (wsReconnectAttempts >= MAX_WS_RECONNECT) { + // Fallback to polling after 5 failed attempts + useWebSocket = false; + updateConnectionStatus('polling'); + startAutoRefresh(); + return; + } + + updateConnectionStatus('connecting'); + const wsUrl = `${protocol}//${window.location.host}/monitor/ws`; + websocket = new WebSocket(wsUrl); + + websocket.onopen = () => { + wsReconnectAttempts = 0; + updateConnectionStatus('connected'); + stopAutoRefresh(); // Stop polling + }; + + websocket.onmessage = (event) => { + const data = JSON.parse(event.data); + updateDashboard(data); // Update all sections + }; + + websocket.onclose = () => { + updateConnectionStatus('disconnected', 'Reconnecting...'); + if (useWebSocket) { + setTimeout(connectWebSocket, 2000 * wsReconnectAttempts); + } else { + startAutoRefresh(); // Fallback to polling + } + }; +} +``` + +**Connection Status Indicator:** + +| Status | Color | Animation | Meaning | +|--------|-------|-----------|---------| +| Live | Green | Pulsing fast | WebSocket connected | +| Connecting... | Yellow | Pulsing slow | Attempting connection | +| Polling | Blue | Pulsing slow | HTTP polling fallback | +| Disconnected | Red | None | Connection failed | + +--- + +## API Layer + +### Request/Response Flow + +``` +Client Request + │ + ▼ +FastAPI Route Handler + │ + ├─→ Monitor: track_request_start() + │ + ├─→ Browser Pool: get_crawler(config) + │ │ + │ ├─→ Check PERMANENT + │ ├─→ Check HOT_POOL + │ ├─→ Check COLD_POOL + │ └─→ Create New (if needed) + │ + ├─→ Execute Crawl + │ │ + │ ├─→ Fetch page + │ ├─→ Extract content + │ ├─→ Apply filters/strategies + │ └─→ Return result + │ + ├─→ Monitor: track_request_end() + │ + └─→ Return Response (browser stays in pool) +``` + +### Error Handling Strategy + +**Levels:** + +1. **Route Level**: HTTP exceptions with proper status codes +2. **Monitor Level**: Try-except with logging, non-critical failures +3. **Pool Level**: Memory checks, lock protection, graceful degradation +4. **WebSocket Level**: Auto-reconnect, fallback to polling + +**Example:** + +```python +@app.post("/crawl") +async def crawl(body: CrawlRequest): + request_id = f"req_{uuid4().hex[:8]}" + + try: + # Monitor tracking (non-blocking on failure) + try: + await get_monitor().track_request_start(...) + except: + pass # Monitor not critical + + # Browser acquisition (with memory protection) + crawler = await get_crawler(browser_config) + + # Crawl execution + result = await crawler.arun(url, config=cfg) + + # Success tracking + try: + await get_monitor().track_request_end(request_id, success=True) + except: + pass + + return result + + except MemoryError as e: + # Memory pressure - return 503 + await get_monitor().track_request_end(request_id, success=False, error=str(e)) + raise HTTPException(503, "Server at capacity") + except Exception as e: + # General errors - return 500 + await get_monitor().track_request_end(request_id, success=False, error=str(e)) + raise HTTPException(500, str(e)) +``` + +--- + +## Memory Management + +### Container Memory Detection + +**Priority Order:** +1. cgroup v2 (`/sys/fs/cgroup/memory.{current,max}`) +2. cgroup v1 (`/sys/fs/cgroup/memory/memory.{usage,limit}_in_bytes`) +3. psutil fallback (may be inaccurate in containers) + +**Usage:** + +```python +mem_pct = get_container_memory_percent() + +if mem_pct >= 95: # Critical + raise MemoryError("Refusing new browser") +elif mem_pct > 80: # High pressure + # Janitor: aggressive cleanup (10s interval, 30s TTL) +elif mem_pct > 60: # Moderate pressure + # Janitor: moderate cleanup (30s interval, 60s TTL) +else: # Normal + # Janitor: relaxed cleanup (60s interval, 300s TTL) +``` + +### Memory Budgets + +| Component | Memory | Notes | +|-----------|--------|-------| +| Base Container | 270 MB | Python + FastAPI + libraries | +| Permanent Browser | 270 MB | Always-on default browser | +| Hot Pool Browser | 180 MB | Per frequently-used config | +| Cold Pool Browser | 180 MB | Per rarely-used config | +| Active Crawl Overhead | 50-200 MB | Temporary, released after request | + +**Example Calculation:** + +``` +Container: 270 MB +Permanent: 270 MB +2x Hot: 360 MB +1x Cold: 180 MB +Total: 1080 MB baseline + +Under load (10 concurrent): ++ Active crawls: ~500-1000 MB += Peak: 1.5-2 GB +``` + +--- + +## Production Optimizations + +### Code Review Fixes Applied + +**Critical (3):** +1. ✅ Lock protection for browser pool access +2. ✅ Async track_janitor_event implementation +3. ✅ Error handling in request tracking + +**Important (8):** +4. ✅ Background persistence worker (replaces fire-and-forget) +5. ✅ Time-based expiry (5min cleanup for old entries) +6. ✅ Input validation (status, limit, metric, window) +7. ✅ Timeline updater timeout (4s max) +8. ✅ Warn when killing browsers with active requests +9. ✅ Monitor cleanup on shutdown +10. ✅ Document memory estimates +11. ✅ Structured error responses (HTTPException) + +### Performance Characteristics + +**Latency:** + +| Scenario | Time | Notes | +|----------|------|-------| +| Pool Hit (Permanent) | <100ms | Browser ready | +| Pool Hit (Hot/Cold) | <100ms | Browser ready | +| New Browser Creation | 3-5s | Chromium startup | +| Simple Page Fetch | 1-3s | Network + render | +| Complex Extraction | 5-10s | LLM processing | + +**Throughput:** + +| Load | Concurrent | Response Time | Success Rate | +|------|-----------|---------------|--------------| +| Light | 1-10 | <3s | 100% | +| Medium | 10-50 | 3-8s | 100% | +| Heavy | 50-100 | 8-15s | 95-100% | +| Extreme | 100+ | 15-30s | 80-95% | + +### Reliability Features + +**Race Condition Protection:** +- `asyncio.Lock` on all pool operations +- Lock on browser pool stats access +- Async janitor event tracking + +**Graceful Degradation:** +- WebSocket → HTTP polling fallback +- Redis persistence failures (logged, non-blocking) +- Monitor tracking failures (logged, non-blocking) + +**Resource Cleanup:** +- Janitor cleanup (adaptive intervals) +- Time-based expiry (5min for old data) +- Shutdown cleanup (persist final stats, close browsers) +- Background worker cancellation + +--- + +## Deployment & Operations + +### Running Locally + +```bash +# Install dependencies +pip install -r requirements.txt + +# Configure +cp .llm.env.example .llm.env +# Edit .llm.env with your API keys + +# Run server +python -m uvicorn server:app --host 0.0.0.0 --port 11235 --reload +``` + +### Docker Deployment + +```bash +# Build image +docker build -t crawl4ai:latest -f Dockerfile . + +# Run container +docker run -d \ + --name crawl4ai \ + -p 11235:11235 \ + --shm-size=1g \ + --env-file .llm.env \ + crawl4ai:latest +``` + +### Production Configuration + +**`config.yml` Key Settings:** + +```yaml +crawler: + browser: + extra_args: + - "--disable-gpu" + - "--disable-dev-shm-usage" + - "--no-sandbox" + kwargs: + headless: true + text_mode: true # Reduces memory by 30-40% + + memory_threshold_percent: 95 # Refuse new browsers above this + + pool: + idle_ttl_sec: 300 # Base TTL for cold pool (5 min) + + rate_limiter: + enabled: true + base_delay: [1.0, 3.0] # Random delay between requests +``` + +### Monitoring + +**Access Dashboard:** +``` +http://localhost:11235/static/monitor/ +``` + +**Check Logs:** +```bash +# All activity +docker logs crawl4ai -f + +# Pool activity only +docker logs crawl4ai | grep -E "(🔥|♨️|❄️|🆕|⬆️)" + +# Errors only +docker logs crawl4ai | grep ERROR +``` + +**Metrics:** +```bash +# Container stats +docker stats crawl4ai + +# Memory percentage +curl http://localhost:11235/monitor/health | jq '.container.memory_percent' + +# Pool status +curl http://localhost:11235/monitor/browsers | jq '.summary' +``` + +--- + +## Troubleshooting & Debugging + +### Common Issues + +**1. WebSocket Not Connecting** + +Symptoms: Yellow "Connecting..." indicator, falls back to blue "Polling" + +Debug: +```bash +# Check server logs +docker logs crawl4ai | grep WebSocket + +# Test WebSocket manually +python test-websocket.py +``` + +Fix: Check firewall/proxy settings, ensure port 11235 accessible + +**2. High Memory Usage** + +Symptoms: Container OOM kills, 503 errors, slow responses + +Debug: +```bash +# Check current memory +curl http://localhost:11235/monitor/health | jq '.container.memory_percent' + +# Check browser pool +curl http://localhost:11235/monitor/browsers + +# Check janitor activity +docker logs crawl4ai | grep "🧹" +``` + +Fix: +- Lower `memory_threshold_percent` in config.yml +- Increase container memory limit +- Enable `text_mode: true` in browser config +- Reduce idle_ttl_sec for more aggressive cleanup + +**3. Browser Pool Not Reusing** + +Symptoms: High "New Created" count, poor reuse rate + +Debug: +```python +# Check config signature matching +from crawl4ai import BrowserConfig +import json, hashlib + +cfg = BrowserConfig(...) # Your config +sig = hashlib.sha1(json.dumps(cfg.to_dict(), sort_keys=True).encode()).hexdigest() +print(f"Config signature: {sig[:8]}") +``` + +Check logs for permanent browser signature: +```bash +docker logs crawl4ai | grep "permanent" +``` + +Fix: Ensure endpoint configs match permanent browser config exactly + +**4. Janitor Not Cleaning Up** + +Symptoms: Memory stays high after idle period + +Debug: +```bash +# Check janitor events +curl http://localhost:11235/monitor/logs/janitor + +# Check pool stats over time +watch -n 5 'curl -s http://localhost:11235/monitor/browsers | jq ".summary"' +``` + +Fix: +- Janitor runs every 10-60s depending on memory +- Hot pool browsers have longer TTL (by design) +- Permanent browser never cleaned (by design) + +### Debug Tools + +**Config Signature Checker:** + +```python +from crawl4ai import BrowserConfig +import json, hashlib + +def check_sig(cfg: BrowserConfig) -> str: + payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",",":")) + sig = hashlib.sha1(payload.encode()).hexdigest() + return sig[:8] + +# Example +cfg1 = BrowserConfig() +cfg2 = BrowserConfig(headless=True) +print(f"Default: {check_sig(cfg1)}") +print(f"Custom: {check_sig(cfg2)}") +``` + +**Monitor Stats Dumper:** + +```bash +#!/bin/bash +# Dump all monitor stats to JSON + +curl -s http://localhost:11235/monitor/health > health.json +curl -s http://localhost:11235/monitor/requests?limit=100 > requests.json +curl -s http://localhost:11235/monitor/browsers > browsers.json +curl -s http://localhost:11235/monitor/logs/janitor > janitor.json +curl -s http://localhost:11235/monitor/logs/errors > errors.json + +echo "Monitor stats dumped to *.json files" +``` + +**WebSocket Test Script:** + +```python +# test-websocket.py (included in repo) +import asyncio +import websockets +import json + +async def test_websocket(): + uri = "ws://localhost:11235/monitor/ws" + async with websockets.connect(uri) as websocket: + for i in range(5): + message = await websocket.recv() + data = json.loads(message) + print(f"\nUpdate #{i+1}:") + print(f" Health: CPU {data['health']['container']['cpu_percent']}%") + print(f" Active Requests: {len(data['requests']['active'])}") + print(f" Browsers: {len(data['browsers'])}") + +asyncio.run(test_websocket()) +``` + +### Performance Tuning + +**For High Throughput:** + +```yaml +# config.yml +crawler: + memory_threshold_percent: 90 # Allow more browsers + pool: + idle_ttl_sec: 600 # Keep browsers longer + rate_limiter: + enabled: false # Disable for max speed +``` + +**For Low Memory:** + +```yaml +# config.yml +crawler: + browser: + kwargs: + text_mode: true # 30-40% memory reduction + memory_threshold_percent: 80 # More conservative + pool: + idle_ttl_sec: 60 # Aggressive cleanup +``` + +**For Stability:** + +```yaml +# config.yml +crawler: + memory_threshold_percent: 85 # Balanced + pool: + idle_ttl_sec: 300 # Moderate cleanup + rate_limiter: + enabled: true + base_delay: [2.0, 5.0] # Prevent rate limiting +``` + +--- + +## Test Suite + +**Location:** `deploy/docker/tests/` + +**Tests:** + +1. `test_1_basic.py` - Health check, container lifecycle +2. `test_2_memory.py` - Memory tracking, leak detection +3. `test_3_pool.py` - Pool reuse validation +4. `test_4_concurrent.py` - Concurrent load testing +5. `test_5_pool_stress.py` - Multi-config pool behavior +6. `test_6_multi_endpoint.py` - All endpoint validation +7. `test_7_cleanup.py` - Janitor cleanup verification + +**Run All Tests:** + +```bash +cd deploy/docker/tests +pip install -r requirements.txt + +# Build image first +cd /path/to/repo +docker build -t crawl4ai-local:latest . + +# Run tests +cd deploy/docker/tests +for test in test_*.py; do + echo "Running $test..." + python $test || break +done +``` + +--- + +## Architecture Decision Log + +### Why 3-Tier Pool? + +**Decision:** PERMANENT + HOT_POOL + COLD_POOL + +**Rationale:** +- 90% of requests use default config → permanent browser serves most traffic +- Frequent variants (hot) deserve longer TTL for better reuse +- Rare configs (cold) should be cleaned aggressively to save memory + +**Alternatives Considered:** +- Single pool: Too simple, no optimization for common case +- LRU cache: Doesn't capture "hot" vs "rare" distinction +- Per-endpoint pools: Too complex, over-engineering + +### Why WebSocket + Polling Fallback? + +**Decision:** WebSocket primary, HTTP polling backup + +**Rationale:** +- WebSocket provides real-time updates (2s interval) +- Polling fallback ensures reliability in restricted networks +- Auto-reconnect handles temporary disconnections + +**Alternatives Considered:** +- Polling only: Works but higher latency, more server load +- WebSocket only: Fails in restricted networks +- Server-Sent Events: One-way, no client messages + +### Why Background Persistence Worker? + +**Decision:** Queue-based worker for Redis operations + +**Rationale:** +- Fire-and-forget loses data on failures +- Queue provides buffering and retry capability +- Non-blocking keeps request path fast + +**Alternatives Considered:** +- Synchronous writes: Blocks request handling +- Fire-and-forget: Silent failures +- Batch writes: Complex state management + +--- + +## Contributing + +When modifying the architecture: + +1. **Maintain backward compatibility** in API contracts +2. **Add tests** for new functionality +3. **Update this document** with architectural changes +4. **Profile memory impact** before production +5. **Test under load** using the test suite + +**Code Review Checklist:** +- [ ] Race conditions protected with locks +- [ ] Error handling with proper logging +- [ ] Graceful degradation on failures +- [ ] Memory impact measured +- [ ] Tests added/updated +- [ ] Documentation updated + +--- + +## License & Credits + +**Crawl4AI** - Created by Unclecode +**GitHub**: https://github.com/unclecode/crawl4ai +**License**: See LICENSE file in repository + +**Architecture & Optimizations**: October 2025 +**WebSocket Monitoring**: October 2025 +**Production Hardening**: October 2025 + +--- + +**End of Technical Architecture Document** + +For questions or issues, please open a GitHub issue at: +https://github.com/unclecode/crawl4ai/issues From 73a5a7b0f589ec17a45dcf6a51fe4de20f6e8b86 Mon Sep 17 00:00:00 2001 From: unclecode Date: Sat, 18 Oct 2025 12:41:29 +0800 Subject: [PATCH 074/119] Update gitignore --- .gitignore | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitignore b/.gitignore index a5389a3e..7fed1b79 100644 --- a/.gitignore +++ b/.gitignore @@ -280,3 +280,14 @@ docs/apps/linkdin/debug*/ docs/apps/linkdin/samples/insights/* scripts/ + + +# Databse files +*.sqlite3 +*.sqlite3-journal +*.db-journal +*.db-wal +*.db-shm +*.db +*.rdb +*.ldb From c7288dd2f1391597c903c2b85377dc6b5ea95621 Mon Sep 17 00:00:00 2001 From: unclecode Date: Sun, 19 Oct 2025 10:24:33 +0800 Subject: [PATCH 075/119] docs: add complete SDK reference documentation Add comprehensive single-page SDK reference combining: - Installation & Setup - Quick Start - Core API (AsyncWebCrawler, arun, arun_many, CrawlResult) - Configuration (BrowserConfig, CrawlerConfig, Parameters) - Crawling Patterns - Content Processing (Markdown, Fit Markdown, Selection, Interaction, Link & Media) - Extraction Strategies (LLM and No-LLM) - Advanced Features (Session Management, Hooks & Auth) Generated using scripts/generate_sdk_docs.py in ultra-dense mode optimized for AI assistant consumption. Stats: 23K words, 185 code blocks, 220KB --- docs/md_v2/complete-sdk-reference.md | 5196 ++++++++++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 5197 insertions(+) create mode 100644 docs/md_v2/complete-sdk-reference.md diff --git a/docs/md_v2/complete-sdk-reference.md b/docs/md_v2/complete-sdk-reference.md new file mode 100644 index 00000000..8a06f65b --- /dev/null +++ b/docs/md_v2/complete-sdk-reference.md @@ -0,0 +1,5196 @@ +# Crawl4AI Complete SDK Documentation + +**Generated:** 2025-10-19 10:23 +**Format:** Ultra-Dense Reference (Optimized for AI Assistants) +**Version:** 0.7.x + +--- + +## Navigation + + +- [Installation & Setup](#installation--setup) +- [Quick Start](#quick-start) +- [Core API](#core-api) +- [Configuration](#configuration) +- [Crawling Patterns](#crawling-patterns) +- [Content Processing](#content-processing) +- [Extraction Strategies](#extraction-strategies) +- [Advanced Features](#advanced-features) + +--- + + +# Installation & Setup + +# Installation & Setup (2023 Edition) +## 1. Basic Installation +```bash +pip install crawl4ai +``` +## 2. Initial Setup & Diagnostics +### 2.1 Run the Setup Command +```bash +crawl4ai-setup +``` +- Performs OS-level checks (e.g., missing libs on Linux) +- Confirms your environment is ready to crawl +### 2.2 Diagnostics +```bash +crawl4ai-doctor +``` +- Check Python version compatibility +- Verify Playwright installation +- Inspect environment variables or library conflicts +If any issues arise, follow its suggestions (e.g., installing additional system packages) and re-run `crawl4ai-setup`. +## 3. Verifying Installation: A Simple Crawl (Skip this step if you already run `crawl4ai-doctor`) +Below is a minimal Python script demonstrating a **basic** crawl. It uses our new **`BrowserConfig`** and **`CrawlerRunConfig`** for clarity, though no custom settings are passed in this example: +```python +import asyncio +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig + +async def main(): + async with AsyncWebCrawler() as crawler: + result = await crawler.arun( + url="https://www.example.com", + ) + print(result.markdown[:300]) # Show the first 300 characters of extracted text + +if __name__ == "__main__": + asyncio.run(main()) +``` +- A headless browser session loads `example.com` +- Crawl4AI returns ~300 characters of markdown. +If errors occur, rerun `crawl4ai-doctor` or manually ensure Playwright is installed correctly. +## 4. Advanced Installation (Optional) +### 4.1 Torch, Transformers, or All +- **Text Clustering (Torch)** + ```bash + pip install crawl4ai[torch] + crawl4ai-setup + ``` +- **Transformers** + ```bash + pip install crawl4ai[transformer] + crawl4ai-setup + ``` +- **All Features** + ```bash + pip install crawl4ai[all] + crawl4ai-setup + ``` +```bash +crawl4ai-download-models +``` +## 5. Docker (Experimental) +```bash +docker pull unclecode/crawl4ai:basic +docker run -p 11235:11235 unclecode/crawl4ai:basic +``` +You can then make POST requests to `http://localhost:11235/crawl` to perform crawls. **Production usage** is discouraged until our new Docker approach is ready (planned in Jan or Feb 2025). +## 6. Local Server Mode (Legacy) +## Summary +1. **Install** with `pip install crawl4ai` and run `crawl4ai-setup`. +2. **Diagnose** with `crawl4ai-doctor` if you see errors. +3. **Verify** by crawling `example.com` with minimal `BrowserConfig` + `CrawlerRunConfig`. + + + +# Quick Start + +# Getting Started with Crawl4AI +1. Run your **first crawl** using minimal configuration. +3. Experiment with a simple **CSS-based extraction** strategy. +5. Crawl a **dynamic** page that loads content via JavaScript. +## 1. Introduction +- An asynchronous crawler, **`AsyncWebCrawler`**. +- Configurable browser and run settings via **`BrowserConfig`** and **`CrawlerRunConfig`**. +- Automatic HTML-to-Markdown conversion via **`DefaultMarkdownGenerator`** (supports optional filters). +- Multiple extraction strategies (LLM-based or “traditional” CSS/XPath-based). +## 2. Your First Crawl +Here’s a minimal Python script that creates an **`AsyncWebCrawler`**, fetches a webpage, and prints the first 300 characters of its Markdown output: +```python +import asyncio +from crawl4ai import AsyncWebCrawler + +async def main(): + async with AsyncWebCrawler() as crawler: + result = await crawler.arun("https://example.com") + print(result.markdown[:300]) # Print first 300 chars + +if __name__ == "__main__": + asyncio.run(main()) +``` +- **`AsyncWebCrawler`** launches a headless browser (Chromium by default). +- It fetches `https://example.com`. +- Crawl4AI automatically converts the HTML into Markdown. +## 3. Basic Configuration (Light Introduction) +1. **`BrowserConfig`**: Controls browser behavior (headless or full UI, user agent, JavaScript toggles, etc.). +2. **`CrawlerRunConfig`**: Controls how each crawl runs (caching, extraction, timeouts, hooking, etc.). +```python +import asyncio +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode + +async def main(): + browser_conf = BrowserConfig(headless=True) # or False to see the browser + run_conf = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS + ) + + async with AsyncWebCrawler(config=browser_conf) as crawler: + result = await crawler.arun( + url="https://example.com", + config=run_conf + ) + print(result.markdown) + +if __name__ == "__main__": + asyncio.run(main()) +``` +> IMPORTANT: By default cache mode is set to `CacheMode.BYPASS` to have fresh content. Set `CacheMode.ENABLED` to enable caching. +## 4. Generating Markdown Output +- **`result.markdown`**: +- **`result.markdown.fit_markdown`**: + The same content after applying any configured **content filter** (e.g., `PruningContentFilter`). +### Example: Using a Filter with `DefaultMarkdownGenerator` +```python +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig +from crawl4ai.content_filter_strategy import PruningContentFilter +from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator + +md_generator = DefaultMarkdownGenerator( + content_filter=PruningContentFilter(threshold=0.4, threshold_type="fixed") +) + +config = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + markdown_generator=md_generator +) + +async with AsyncWebCrawler() as crawler: + result = await crawler.arun("https://news.ycombinator.com", config=config) + print("Raw Markdown length:", len(result.markdown.raw_markdown)) + print("Fit Markdown length:", len(result.markdown.fit_markdown)) +``` +**Note**: If you do **not** specify a content filter or markdown generator, you’ll typically see only the raw Markdown. `PruningContentFilter` may adds around `50ms` in processing time. We’ll dive deeper into these strategies in a dedicated **Markdown Generation** tutorial. +## 5. Simple Data Extraction (CSS-based) +```python +from crawl4ai import JsonCssExtractionStrategy +from crawl4ai import LLMConfig + +# Generate a schema (one-time cost) +html = "

Gaming Laptop

$999.99
" + +# Using OpenAI (requires API token) +schema = JsonCssExtractionStrategy.generate_schema( + html, + llm_config = LLMConfig(provider="openai/gpt-4o",api_token="your-openai-token") # Required for OpenAI +) + +# Or using Ollama (open source, no token needed) +schema = JsonCssExtractionStrategy.generate_schema( + html, + llm_config = LLMConfig(provider="ollama/llama3.3", api_token=None) # Not needed for Ollama +) + +# Use the schema for fast, repeated extractions +strategy = JsonCssExtractionStrategy(schema) +``` +```python +import asyncio +import json +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode +from crawl4ai import JsonCssExtractionStrategy + +async def main(): + schema = { + "name": "Example Items", + "baseSelector": "div.item", + "fields": [ + {"name": "title", "selector": "h2", "type": "text"}, + {"name": "link", "selector": "a", "type": "attribute", "attribute": "href"} + ] + } + + raw_html = "

Item 1

Link 1
" + + async with AsyncWebCrawler() as crawler: + result = await crawler.arun( + url="raw://" + raw_html, + config=CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + extraction_strategy=JsonCssExtractionStrategy(schema) + ) + ) + # The JSON output is stored in 'extracted_content' + data = json.loads(result.extracted_content) + print(data) + +if __name__ == "__main__": + asyncio.run(main()) +``` +- Great for repetitive page structures (e.g., item listings, articles). +- No AI usage or costs. +- The crawler returns a JSON string you can parse or store. +> Tips: You can pass raw HTML to the crawler instead of a URL. To do so, prefix the HTML with `raw://`. +## 6. Simple Data Extraction (LLM-based) +- **Open-Source Models** (e.g., `ollama/llama3.3`, `no_token`) +- **OpenAI Models** (e.g., `openai/gpt-4`, requires `api_token`) +- Or any provider supported by the underlying library +```python +import os +import json +import asyncio +from pydantic import BaseModel, Field +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, LLMConfig +from crawl4ai import LLMExtractionStrategy + +class OpenAIModelFee(BaseModel): + model_name: str = Field(..., description="Name of the OpenAI model.") + input_fee: str = Field(..., description="Fee for input token for the OpenAI model.") + output_fee: str = Field( + ..., description="Fee for output token for the OpenAI model." + ) + +async def extract_structured_data_using_llm( + provider: str, api_token: str = None, extra_headers: Dict[str, str] = None +): + print(f"\n--- Extracting Structured Data with {provider} ---") + + if api_token is None and provider != "ollama": + print(f"API token is required for {provider}. Skipping this example.") + return + + browser_config = BrowserConfig(headless=True) + + extra_args = {"temperature": 0, "top_p": 0.9, "max_tokens": 2000} + if extra_headers: + extra_args["extra_headers"] = extra_headers + + crawler_config = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + word_count_threshold=1, + page_timeout=80000, + extraction_strategy=LLMExtractionStrategy( + llm_config = LLMConfig(provider=provider,api_token=api_token), + schema=OpenAIModelFee.model_json_schema(), + extraction_type="schema", + instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. + Do not miss any models in the entire content.""", + extra_args=extra_args, + ), + ) + + async with AsyncWebCrawler(config=browser_config) as crawler: + result = await crawler.arun( + url="https://openai.com/api/pricing/", config=crawler_config + ) + print(result.extracted_content) + +if __name__ == "__main__": + + asyncio.run( + extract_structured_data_using_llm( + provider="openai/gpt-4o", api_token=os.getenv("OPENAI_API_KEY") + ) + ) +``` +- We define a Pydantic schema (`PricingInfo`) describing the fields we want. +## 7. Adaptive Crawling (New!) +```python +import asyncio +from crawl4ai import AsyncWebCrawler, AdaptiveCrawler + +async def adaptive_example(): + async with AsyncWebCrawler() as crawler: + adaptive = AdaptiveCrawler(crawler) + + # Start adaptive crawling + result = await adaptive.digest( + start_url="https://docs.python.org/3/", + query="async context managers" + ) + + # View results + adaptive.print_stats() + print(f"Crawled {len(result.crawled_urls)} pages") + print(f"Achieved {adaptive.confidence:.0%} confidence") + +if __name__ == "__main__": + asyncio.run(adaptive_example()) +``` +- **Automatic stopping**: Stops when sufficient information is gathered +- **Intelligent link selection**: Follows only relevant links +- **Confidence scoring**: Know how complete your information is +## 8. Multi-URL Concurrency (Preview) +If you need to crawl multiple URLs in **parallel**, you can use `arun_many()`. By default, Crawl4AI employs a **MemoryAdaptiveDispatcher**, automatically adjusting concurrency based on system resources. Here’s a quick glimpse: +```python +import asyncio +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode + +async def quick_parallel_example(): + urls = [ + "https://example.com/page1", + "https://example.com/page2", + "https://example.com/page3" + ] + + run_conf = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + stream=True # Enable streaming mode + ) + + async with AsyncWebCrawler() as crawler: + # Stream results as they complete + async for result in await crawler.arun_many(urls, config=run_conf): + if result.success: + print(f"[OK] {result.url}, length: {len(result.markdown.raw_markdown)}") + else: + print(f"[ERROR] {result.url} => {result.error_message}") + + # Or get all results at once (default behavior) + run_conf = run_conf.clone(stream=False) + results = await crawler.arun_many(urls, config=run_conf) + for res in results: + if res.success: + print(f"[OK] {res.url}, length: {len(res.markdown.raw_markdown)}") + else: + print(f"[ERROR] {res.url} => {res.error_message}") + +if __name__ == "__main__": + asyncio.run(quick_parallel_example()) +``` +1. **Streaming mode** (`stream=True`): Process results as they become available using `async for` +2. **Batch mode** (`stream=False`): Wait for all results to complete +## 8. Dynamic Content Example +Some sites require multiple “page clicks” or dynamic JavaScript updates. Below is an example showing how to **click** a “Next Page” button and wait for new commits to load on GitHub, using **`BrowserConfig`** and **`CrawlerRunConfig`**: +```python +import asyncio +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode +from crawl4ai import JsonCssExtractionStrategy + +async def extract_structured_data_using_css_extractor(): + print("\n--- Using JsonCssExtractionStrategy for Fast Structured Output ---") + schema = { + "name": "KidoCode Courses", + "baseSelector": "section.charge-methodology .w-tab-content > div", + "fields": [ + { + "name": "section_title", + "selector": "h3.heading-50", + "type": "text", + }, + { + "name": "section_description", + "selector": ".charge-content", + "type": "text", + }, + { + "name": "course_name", + "selector": ".text-block-93", + "type": "text", + }, + { + "name": "course_description", + "selector": ".course-content-text", + "type": "text", + }, + { + "name": "course_icon", + "selector": ".image-92", + "type": "attribute", + "attribute": "src", + }, + ], + } + + browser_config = BrowserConfig(headless=True, java_script_enabled=True) + + js_click_tabs = """ + (async () => { + const tabs = document.querySelectorAll("section.charge-methodology .tabs-menu-3 > div"); + for(let tab of tabs) { + tab.scrollIntoView(); + tab.click(); + await new Promise(r => setTimeout(r, 500)); + } + })(); + """ + + crawler_config = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + extraction_strategy=JsonCssExtractionStrategy(schema), + js_code=[js_click_tabs], + ) + + async with AsyncWebCrawler(config=browser_config) as crawler: + result = await crawler.arun( + url="https://www.kidocode.com/degrees/technology", config=crawler_config + ) + + companies = json.loads(result.extracted_content) + print(f"Successfully extracted {len(companies)} companies") + print(json.dumps(companies[0], indent=2)) + +async def main(): + await extract_structured_data_using_css_extractor() + +if __name__ == "__main__": + asyncio.run(main()) +``` +- **`BrowserConfig(headless=False)`**: We want to watch it click “Next Page.” +- **`CrawlerRunConfig(...)`**: We specify the extraction strategy, pass `session_id` to reuse the same page. +- **`js_code`** and **`wait_for`** are used for subsequent pages (`page > 0`) to click the “Next” button and wait for new commits to load. +- **`js_only=True`** indicates we’re not re-navigating but continuing the existing session. +- Finally, we call `kill_session()` to clean up the page and browser session. +## 9. Next Steps +1. Performed a basic crawl and printed Markdown. +2. Used **content filters** with a markdown generator. +3. Extracted JSON via **CSS** or **LLM** strategies. +4. Handled **dynamic** pages with JavaScript triggers. + + + +# Core API + +# AsyncWebCrawler +The **`AsyncWebCrawler`** is the core class for asynchronous web crawling in Crawl4AI. You typically create it **once**, optionally customize it with a **`BrowserConfig`** (e.g., headless, user agent), then **run** multiple **`arun()`** calls with different **`CrawlerRunConfig`** objects. +1. **Create** a `BrowserConfig` for global browser settings.  +2. **Instantiate** `AsyncWebCrawler(config=browser_config)`.  +3. **Use** the crawler in an async context manager (`async with`) or manage start/close manually.  +4. **Call** `arun(url, config=crawler_run_config)` for each page you want. +## 1. Constructor Overview +```python +class AsyncWebCrawler: + def __init__( + self, + crawler_strategy: Optional[AsyncCrawlerStrategy] = None, + config: Optional[BrowserConfig] = None, + always_bypass_cache: bool = False, # deprecated + always_by_pass_cache: Optional[bool] = None, # also deprecated + base_directory: str = ..., + thread_safe: bool = False, + **kwargs, + ): + """ + Create an AsyncWebCrawler instance. + + Args: + crawler_strategy: + (Advanced) Provide a custom crawler strategy if needed. + config: + A BrowserConfig object specifying how the browser is set up. + always_bypass_cache: + (Deprecated) Use CrawlerRunConfig.cache_mode instead. + base_directory: + Folder for storing caches/logs (if relevant). + thread_safe: + If True, attempts some concurrency safeguards. Usually False. + **kwargs: + Additional legacy or debugging parameters. + """ + ) + +### Typical Initialization + +```python +from crawl4ai import AsyncWebCrawler, BrowserConfig +browser_cfg = BrowserConfig( + browser_type="chromium", + headless=True, + verbose=True +crawler = AsyncWebCrawler(config=browser_cfg) +``` + +**Notes**: + +- **Legacy** parameters like `always_bypass_cache` remain for backward compatibility, but prefer to set **caching** in `CrawlerRunConfig`. + +--- + +## 2. Lifecycle: Start/Close or Context Manager + +### 2.1 Context Manager (Recommended) + +```python +async with AsyncWebCrawler(config=browser_cfg) as crawler: + result = await crawler.arun("https://example.com") + # The crawler automatically starts/closes resources +``` + +When the `async with` block ends, the crawler cleans up (closes the browser, etc.). + +### 2.2 Manual Start & Close + +```python +crawler = AsyncWebCrawler(config=browser_cfg) +await crawler.start() +result1 = await crawler.arun("https://example.com") +result2 = await crawler.arun("https://another.com") +await crawler.close() +``` + +Use this style if you have a **long-running** application or need full control of the crawler’s lifecycle. + +--- + +## 3. Primary Method: `arun()` + +```python +async def arun( + url: str, + config: Optional[CrawlerRunConfig] = None, + # Legacy parameters for backward compatibility... +``` + +### 3.1 New Approach + +You pass a `CrawlerRunConfig` object that sets up everything about a crawl—content filtering, caching, session reuse, JS code, screenshots, etc. + +```python +import asyncio +from crawl4ai import CrawlerRunConfig, CacheMode +run_cfg = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + css_selector="main.article", + word_count_threshold=10, + screenshot=True +async with AsyncWebCrawler(config=browser_cfg) as crawler: + result = await crawler.arun("https://example.com/news", config=run_cfg) +``` + +### 3.2 Legacy Parameters Still Accepted + +For **backward** compatibility, `arun()` can still accept direct arguments like `css_selector=...`, `word_count_threshold=...`, etc., but we strongly advise migrating them into a **`CrawlerRunConfig`**. + +--- + +## 4. Batch Processing: `arun_many()` + +```python +async def arun_many( + urls: List[str], + config: Optional[CrawlerRunConfig] = None, + # Legacy parameters maintained for backwards compatibility... +``` + +### 4.1 Resource-Aware Crawling + +The `arun_many()` method now uses an intelligent dispatcher that: + +- Monitors system memory usage +- Implements adaptive rate limiting +- Provides detailed progress monitoring +- Manages concurrent crawls efficiently + +### 4.2 Example Usage + +Check page [Multi-url Crawling](../advanced/multi-url-crawling.md) for a detailed example of how to use `arun_many()`. + +```python +### 4.3 Key Features +1. **Rate Limiting** + - Automatic delay between requests + - Exponential backoff on rate limit detection + - Domain-specific rate limiting + - Configurable retry strategy +2. **Resource Monitoring** + - Memory usage tracking + - Adaptive concurrency based on system load + - Automatic pausing when resources are constrained +3. **Progress Monitoring** + - Detailed or aggregated progress display + - Real-time status updates + - Memory usage statistics +4. **Error Handling** + - Graceful handling of rate limits + - Automatic retries with backoff + - Detailed error reporting +## 5. `CrawlResult` Output +Each `arun()` returns a **`CrawlResult`** containing: +- `url`: Final URL (if redirected). +- `html`: Original HTML. +- `cleaned_html`: Sanitized HTML. +- `markdown_v2`: Deprecated. Instead just use regular `markdown` +- `extracted_content`: If an extraction strategy was used (JSON for CSS/LLM strategies). +- `screenshot`, `pdf`: If screenshots/PDF requested. +- `media`, `links`: Information about discovered images/links. +- `success`, `error_message`: Status info. +## 6. Quick Example +```python +import asyncio +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode +from crawl4ai import JsonCssExtractionStrategy +import json + +async def main(): + # 1. Browser config + browser_cfg = BrowserConfig( + browser_type="firefox", + headless=False, + verbose=True + ) + + # 2. Run config + schema = { + "name": "Articles", + "baseSelector": "article.post", + "fields": [ + { + "name": "title", + "selector": "h2", + "type": "text" + }, + { + "name": "url", + "selector": "a", + "type": "attribute", + "attribute": "href" + } + ] + } + + run_cfg = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + extraction_strategy=JsonCssExtractionStrategy(schema), + word_count_threshold=15, + remove_overlay_elements=True, + wait_for="css:.post" # Wait for posts to appear + ) + + async with AsyncWebCrawler(config=browser_cfg) as crawler: + result = await crawler.arun( + url="https://example.com/blog", + config=run_cfg + ) + + if result.success: + print("Cleaned HTML length:", len(result.cleaned_html)) + if result.extracted_content: + articles = json.loads(result.extracted_content) + print("Extracted articles:", articles[:2]) + else: + print("Error:", result.error_message) + +asyncio.run(main()) +``` +- We define a **`BrowserConfig`** with Firefox, no headless, and `verbose=True`.  +- We define a **`CrawlerRunConfig`** that **bypasses cache**, uses a **CSS** extraction schema, has a `word_count_threshold=15`, etc.  +- We pass them to `AsyncWebCrawler(config=...)` and `arun(url=..., config=...)`. +## 7. Best Practices & Migration Notes +1. **Use** `BrowserConfig` for **global** settings about the browser’s environment.  +2. **Use** `CrawlerRunConfig` for **per-crawl** logic (caching, content filtering, extraction strategies, wait conditions).  +3. **Avoid** legacy parameters like `css_selector` or `word_count_threshold` directly in `arun()`. Instead: + ```python + run_cfg = CrawlerRunConfig(css_selector=".main-content", word_count_threshold=20) + result = await crawler.arun(url="...", config=run_cfg) + ``` +## 8. Summary +- **Constructor** accepts **`BrowserConfig`** (or defaults).  +- **`arun(url, config=CrawlerRunConfig)`** is the main method for single-page crawls.  +- **`arun_many(urls, config=CrawlerRunConfig)`** handles concurrency across multiple URLs.  +- For advanced lifecycle control, use `start()` and `close()` explicitly.  +- If you used `AsyncWebCrawler(browser_type="chromium", css_selector="...")`, move browser settings to `BrowserConfig(...)` and content/crawl logic to `CrawlerRunConfig(...)`. + + +# `arun()` Parameter Guide (New Approach) +In Crawl4AI’s **latest** configuration model, nearly all parameters that once went directly to `arun()` are now part of **`CrawlerRunConfig`**. When calling `arun()`, you provide: +```python +await crawler.arun( + url="https://example.com", + config=my_run_config +) +``` +Below is an organized look at the parameters that can go inside `CrawlerRunConfig`, divided by their functional areas. For **Browser** settings (e.g., `headless`, `browser_type`), see [BrowserConfig](./parameters.md). +## 1. Core Usage +```python +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode + +async def main(): + run_config = CrawlerRunConfig( + verbose=True, # Detailed logging + cache_mode=CacheMode.ENABLED, # Use normal read/write cache + check_robots_txt=True, # Respect robots.txt rules + # ... other parameters + ) + + async with AsyncWebCrawler() as crawler: + result = await crawler.arun( + url="https://example.com", + config=run_config + ) + + # Check if blocked by robots.txt + if not result.success and result.status_code == 403: + print(f"Error: {result.error_message}") +``` +- `verbose=True` logs each crawl step.  +- `cache_mode` decides how to read/write the local crawl cache. +## 2. Cache Control +**`cache_mode`** (default: `CacheMode.ENABLED`) +Use a built-in enum from `CacheMode`: +- `ENABLED`: Normal caching—reads if available, writes if missing. +- `DISABLED`: No caching—always refetch pages. +- `READ_ONLY`: Reads from cache only; no new writes. +- `WRITE_ONLY`: Writes to cache but doesn’t read existing data. +- `BYPASS`: Skips reading cache for this crawl (though it might still write if set up that way). +```python +run_config = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS +) +``` +- `bypass_cache=True` acts like `CacheMode.BYPASS`. +- `disable_cache=True` acts like `CacheMode.DISABLED`. +- `no_cache_read=True` acts like `CacheMode.WRITE_ONLY`. +- `no_cache_write=True` acts like `CacheMode.READ_ONLY`. +## 3. Content Processing & Selection +### 3.1 Text Processing +```python +run_config = CrawlerRunConfig( + word_count_threshold=10, # Ignore text blocks <10 words + only_text=False, # If True, tries to remove non-text elements + keep_data_attributes=False # Keep or discard data-* attributes +) +``` +### 3.2 Content Selection +```python +run_config = CrawlerRunConfig( + css_selector=".main-content", # Focus on .main-content region only + excluded_tags=["form", "nav"], # Remove entire tag blocks + remove_forms=True, # Specifically strip
elements + remove_overlay_elements=True, # Attempt to remove modals/popups +) +``` +### 3.3 Link Handling +```python +run_config = CrawlerRunConfig( + exclude_external_links=True, # Remove external links from final content + exclude_social_media_links=True, # Remove links to known social sites + exclude_domains=["ads.example.com"], # Exclude links to these domains + exclude_social_media_domains=["facebook.com","twitter.com"], # Extend the default list +) +``` +### 3.4 Media Filtering +```python +run_config = CrawlerRunConfig( + exclude_external_images=True # Strip images from other domains +) +``` +## 4. Page Navigation & Timing +### 4.1 Basic Browser Flow +```python +run_config = CrawlerRunConfig( + wait_for="css:.dynamic-content", # Wait for .dynamic-content + delay_before_return_html=2.0, # Wait 2s before capturing final HTML + page_timeout=60000, # Navigation & script timeout (ms) +) +``` +- `wait_for`: + - `"css:selector"` or + - `"js:() => boolean"` + e.g. `js:() => document.querySelectorAll('.item').length > 10`. +- `mean_delay` & `max_range`: define random delays for `arun_many()` calls.  +- `semaphore_count`: concurrency limit when crawling multiple URLs. +### 4.2 JavaScript Execution +```python +run_config = CrawlerRunConfig( + js_code=[ + "window.scrollTo(0, document.body.scrollHeight);", + "document.querySelector('.load-more')?.click();" + ], + js_only=False +) +``` +- `js_code` can be a single string or a list of strings.  +- `js_only=True` means “I’m continuing in the same session with new JS steps, no new full navigation.” +### 4.3 Anti-Bot +```python +run_config = CrawlerRunConfig( + magic=True, + simulate_user=True, + override_navigator=True +) +``` +- `magic=True` tries multiple stealth features.  +- `simulate_user=True` mimics mouse movements or random delays.  +- `override_navigator=True` fakes some navigator properties (like user agent checks). +## 5. Session Management +**`session_id`**: +```python +run_config = CrawlerRunConfig( + session_id="my_session123" +) +``` +If re-used in subsequent `arun()` calls, the same tab/page context is continued (helpful for multi-step tasks or stateful browsing). +## 6. Screenshot, PDF & Media Options +```python +run_config = CrawlerRunConfig( + screenshot=True, # Grab a screenshot as base64 + screenshot_wait_for=1.0, # Wait 1s before capturing + pdf=True, # Also produce a PDF + image_description_min_word_threshold=5, # If analyzing alt text + image_score_threshold=3, # Filter out low-score images +) +``` +- `result.screenshot` → Base64 screenshot string. +- `result.pdf` → Byte array with PDF data. +## 7. Extraction Strategy +**For advanced data extraction** (CSS/LLM-based), set `extraction_strategy`: +```python +run_config = CrawlerRunConfig( + extraction_strategy=my_css_or_llm_strategy +) +``` +The extracted data will appear in `result.extracted_content`. +## 8. Comprehensive Example +Below is a snippet combining many parameters: +```python +import asyncio +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode +from crawl4ai import JsonCssExtractionStrategy + +async def main(): + # Example schema + schema = { + "name": "Articles", + "baseSelector": "article.post", + "fields": [ + {"name": "title", "selector": "h2", "type": "text"}, + {"name": "link", "selector": "a", "type": "attribute", "attribute": "href"} + ] + } + + run_config = CrawlerRunConfig( + # Core + verbose=True, + cache_mode=CacheMode.ENABLED, + check_robots_txt=True, # Respect robots.txt rules + + # Content + word_count_threshold=10, + css_selector="main.content", + excluded_tags=["nav", "footer"], + exclude_external_links=True, + + # Page & JS + js_code="document.querySelector('.show-more')?.click();", + wait_for="css:.loaded-block", + page_timeout=30000, + + # Extraction + extraction_strategy=JsonCssExtractionStrategy(schema), + + # Session + session_id="persistent_session", + + # Media + screenshot=True, + pdf=True, + + # Anti-bot + simulate_user=True, + magic=True, + ) + + async with AsyncWebCrawler() as crawler: + result = await crawler.arun("https://example.com/posts", config=run_config) + if result.success: + print("HTML length:", len(result.cleaned_html)) + print("Extraction JSON:", result.extracted_content) + if result.screenshot: + print("Screenshot length:", len(result.screenshot)) + if result.pdf: + print("PDF bytes length:", len(result.pdf)) + else: + print("Error:", result.error_message) + +if __name__ == "__main__": + asyncio.run(main()) +``` +1. **Crawling** the main content region, ignoring external links.  +2. Running **JavaScript** to click “.show-more”.  +3. **Waiting** for “.loaded-block” to appear.  +4. Generating a **screenshot** & **PDF** of the final page.  +## 9. Best Practices +1. **Use `BrowserConfig` for global browser** settings (headless, user agent).  +2. **Use `CrawlerRunConfig`** to handle the **specific** crawl needs: content filtering, caching, JS, screenshot, extraction, etc.  +4. **Limit** large concurrency (`semaphore_count`) if the site or your system can’t handle it.  +5. For dynamic pages, set `js_code` or `scan_full_page` so you load all content. +## 10. Conclusion +All parameters that used to be direct arguments to `arun()` now belong in **`CrawlerRunConfig`**. This approach: +- Makes code **clearer** and **more maintainable**.  + + +# `arun_many(...)` Reference +> **Note**: This function is very similar to [`arun()`](./arun.md) but focused on **concurrent** or **batch** crawling. If you’re unfamiliar with `arun()` usage, please read that doc first, then review this for differences. +## Function Signature +```python +async def arun_many( + urls: Union[List[str], List[Any]], + config: Optional[Union[CrawlerRunConfig, List[CrawlerRunConfig]]] = None, + dispatcher: Optional[BaseDispatcher] = None, + ... +) -> Union[List[CrawlResult], AsyncGenerator[CrawlResult, None]]: + """ + Crawl multiple URLs concurrently or in batches. + + :param urls: A list of URLs (or tasks) to crawl. + :param config: (Optional) Either: + - A single `CrawlerRunConfig` applying to all URLs + - A list of `CrawlerRunConfig` objects with url_matcher patterns + :param dispatcher: (Optional) A concurrency controller (e.g. MemoryAdaptiveDispatcher). + ... + :return: Either a list of `CrawlResult` objects, or an async generator if streaming is enabled. + """ +``` +## Differences from `arun()` +1. **Multiple URLs**: + - Instead of crawling a single URL, you pass a list of them (strings or tasks).  + - The function returns either a **list** of `CrawlResult` or an **async generator** if streaming is enabled. +2. **Concurrency & Dispatchers**: + - **`dispatcher`** param allows advanced concurrency control.  + - If omitted, a default dispatcher (like `MemoryAdaptiveDispatcher`) is used internally.  +3. **Streaming Support**: + - Enable streaming by setting `stream=True` in your `CrawlerRunConfig`. + - When streaming, use `async for` to process results as they become available. +4. **Parallel** Execution**: + - `arun_many()` can run multiple requests concurrently under the hood.  + - Each `CrawlResult` might also include a **`dispatch_result`** with concurrency details (like memory usage, start/end times). +### Basic Example (Batch Mode) +```python +# Minimal usage: The default dispatcher will be used +results = await crawler.arun_many( + urls=["https://site1.com", "https://site2.com"], + config=CrawlerRunConfig(stream=False) # Default behavior +) + +for res in results: + if res.success: + print(res.url, "crawled OK!") + else: + print("Failed:", res.url, "-", res.error_message) +``` +### Streaming Example +```python +config = CrawlerRunConfig( + stream=True, # Enable streaming mode + cache_mode=CacheMode.BYPASS +) + +# Process results as they complete +async for result in await crawler.arun_many( + urls=["https://site1.com", "https://site2.com", "https://site3.com"], + config=config +): + if result.success: + print(f"Just completed: {result.url}") + # Process each result immediately + process_result(result) +``` +### With a Custom Dispatcher +```python +dispatcher = MemoryAdaptiveDispatcher( + memory_threshold_percent=70.0, + max_session_permit=10 +) +results = await crawler.arun_many( + urls=["https://site1.com", "https://site2.com", "https://site3.com"], + config=my_run_config, + dispatcher=dispatcher +) +``` +### URL-Specific Configurations +Instead of using one config for all URLs, provide a list of configs with `url_matcher` patterns: +```python +from crawl4ai import CrawlerRunConfig, MatchMode +from crawl4ai.processors.pdf import PDFContentScrapingStrategy +from crawl4ai.extraction_strategy import JsonCssExtractionStrategy +from crawl4ai.content_filter_strategy import PruningContentFilter +from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator + +# PDF files - specialized extraction +pdf_config = CrawlerRunConfig( + url_matcher="*.pdf", + scraping_strategy=PDFContentScrapingStrategy() +) + +# Blog/article pages - content filtering +blog_config = CrawlerRunConfig( + url_matcher=["*/blog/*", "*/article/*", "*python.org*"], + markdown_generator=DefaultMarkdownGenerator( + content_filter=PruningContentFilter(threshold=0.48) + ) +) + +# Dynamic pages - JavaScript execution +github_config = CrawlerRunConfig( + url_matcher=lambda url: 'github.com' in url, + js_code="window.scrollTo(0, 500);" +) + +# API endpoints - JSON extraction +api_config = CrawlerRunConfig( + url_matcher=lambda url: 'api' in url or url.endswith('.json'), + # Custome settings for JSON extraction +) + +# Default fallback config +default_config = CrawlerRunConfig() # No url_matcher means it never matches except as fallback + +# Pass the list of configs - first match wins! +results = await crawler.arun_many( + urls=[ + "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf", # → pdf_config + "https://blog.python.org/", # → blog_config + "https://github.com/microsoft/playwright", # → github_config + "https://httpbin.org/json", # → api_config + "https://example.com/" # → default_config + ], + config=[pdf_config, blog_config, github_config, api_config, default_config] +) +``` +- **String patterns**: `"*.pdf"`, `"*/blog/*"`, `"*python.org*"` +- **Function matchers**: `lambda url: 'api' in url` +- **Mixed patterns**: Combine strings and functions with `MatchMode.OR` or `MatchMode.AND` +- **First match wins**: Configs are evaluated in order +- `dispatch_result` in each `CrawlResult` (if using concurrency) can hold memory and timing info.  +- **Important**: Always include a default config (without `url_matcher`) as the last item if you want to handle all URLs. Otherwise, unmatched URLs will fail. +### Return Value +Either a **list** of [`CrawlResult`](./crawl-result.md) objects, or an **async generator** if streaming is enabled. You can iterate to check `result.success` or read each item’s `extracted_content`, `markdown`, or `dispatch_result`. +## Dispatcher Reference +- **`MemoryAdaptiveDispatcher`**: Dynamically manages concurrency based on system memory usage.  +- **`SemaphoreDispatcher`**: Fixed concurrency limit, simpler but less adaptive.  +## Common Pitfalls +3. **Error Handling**: Each `CrawlResult` might fail for different reasons—always check `result.success` or the `error_message` before proceeding. +## Conclusion +Use `arun_many()` when you want to **crawl multiple URLs** simultaneously or in controlled parallel tasks. If you need advanced concurrency features (like memory-based adaptive throttling or complex rate-limiting), provide a **dispatcher**. Each result is a standard `CrawlResult`, possibly augmented with concurrency stats (`dispatch_result`) for deeper inspection. For more details on concurrency logic and dispatchers, see the [Advanced Multi-URL Crawling](../advanced/multi-url-crawling.md) docs. + + +# `CrawlResult` Reference +The **`CrawlResult`** class encapsulates everything returned after a single crawl operation. It provides the **raw or processed content**, details on links and media, plus optional metadata (like screenshots, PDFs, or extracted JSON). +**Location**: `crawl4ai/crawler/models.py` (for reference) +```python +class CrawlResult(BaseModel): + url: str + html: str + success: bool + cleaned_html: Optional[str] = None + fit_html: Optional[str] = None # Preprocessed HTML optimized for extraction + media: Dict[str, List[Dict]] = {} + links: Dict[str, List[Dict]] = {} + downloaded_files: Optional[List[str]] = None + screenshot: Optional[str] = None + pdf : Optional[bytes] = None + mhtml: Optional[str] = None + markdown: Optional[Union[str, MarkdownGenerationResult]] = None + extracted_content: Optional[str] = None + metadata: Optional[dict] = None + error_message: Optional[str] = None + session_id: Optional[str] = None + response_headers: Optional[dict] = None + status_code: Optional[int] = None + ssl_certificate: Optional[SSLCertificate] = None + dispatch_result: Optional[DispatchResult] = None + ... +``` +## 1. Basic Crawl Info +### 1.1 **`url`** *(str)* +```python +print(result.url) # e.g., "https://example.com/" +``` +### 1.2 **`success`** *(bool)* +**What**: `True` if the crawl pipeline ended without major errors; `False` otherwise. +```python +if not result.success: + print(f"Crawl failed: {result.error_message}") +``` +### 1.3 **`status_code`** *(Optional[int])* +```python +if result.status_code == 404: + print("Page not found!") +``` +### 1.4 **`error_message`** *(Optional[str])* +**What**: If `success=False`, a textual description of the failure. +```python +if not result.success: + print("Error:", result.error_message) +``` +### 1.5 **`session_id`** *(Optional[str])* +```python +# If you used session_id="login_session" in CrawlerRunConfig, see it here: +print("Session:", result.session_id) +``` +### 1.6 **`response_headers`** *(Optional[dict])* +```python +if result.response_headers: + print("Server:", result.response_headers.get("Server", "Unknown")) +``` +### 1.7 **`ssl_certificate`** *(Optional[SSLCertificate])* +**What**: If `fetch_ssl_certificate=True` in your CrawlerRunConfig, **`result.ssl_certificate`** contains a [**`SSLCertificate`**](../advanced/ssl-certificate.md) object describing the site's certificate. You can export the cert in multiple formats (PEM/DER/JSON) or access its properties like `issuer`, + `subject`, `valid_from`, `valid_until`, etc. +```python +if result.ssl_certificate: + print("Issuer:", result.ssl_certificate.issuer) +``` +## 2. Raw / Cleaned Content +### 2.1 **`html`** *(str)* +```python +# Possibly large +print(len(result.html)) +``` +### 2.2 **`cleaned_html`** *(Optional[str])* +**What**: A sanitized HTML version—scripts, styles, or excluded tags are removed based on your `CrawlerRunConfig`. +```python +print(result.cleaned_html[:500]) # Show a snippet +``` +## 3. Markdown Fields +### 3.1 The Markdown Generation Approach +- **Raw** markdown +- **Links as citations** (with a references section) +- **Fit** markdown if a **content filter** is used (like Pruning or BM25) +**`MarkdownGenerationResult`** includes: +- **`raw_markdown`** *(str)*: The full HTML→Markdown conversion. +- **`markdown_with_citations`** *(str)*: Same markdown, but with link references as academic-style citations. +- **`references_markdown`** *(str)*: The reference list or footnotes at the end. +- **`fit_markdown`** *(Optional[str])*: If content filtering (Pruning/BM25) was applied, the filtered "fit" text. +- **`fit_html`** *(Optional[str])*: The HTML that led to `fit_markdown`. +```python +if result.markdown: + md_res = result.markdown + print("Raw MD:", md_res.raw_markdown[:300]) + print("Citations MD:", md_res.markdown_with_citations[:300]) + print("References:", md_res.references_markdown) + if md_res.fit_markdown: + print("Pruned text:", md_res.fit_markdown[:300]) +``` +### 3.2 **`markdown`** *(Optional[Union[str, MarkdownGenerationResult]])* +**What**: Holds the `MarkdownGenerationResult`. +```python +print(result.markdown.raw_markdown[:200]) +print(result.markdown.fit_markdown) +print(result.markdown.fit_html) +``` +**Important**: "Fit" content (in `fit_markdown`/`fit_html`) exists in result.markdown, only if you used a **filter** (like **PruningContentFilter** or **BM25ContentFilter**) within a `MarkdownGenerationStrategy`. +## 4. Media & Links +### 4.1 **`media`** *(Dict[str, List[Dict]])* +**What**: Contains info about discovered images, videos, or audio. Typically keys: `"images"`, `"videos"`, `"audios"`. +- `src` *(str)*: Media URL +- `alt` or `title` *(str)*: Descriptive text +- `score` *(float)*: Relevance score if the crawler's heuristic found it "important" +- `desc` or `description` *(Optional[str])*: Additional context extracted from surrounding text +```python +images = result.media.get("images", []) +for img in images: + if img.get("score", 0) > 5: + print("High-value image:", img["src"]) +``` +### 4.2 **`links`** *(Dict[str, List[Dict]])* +**What**: Holds internal and external link data. Usually two keys: `"internal"` and `"external"`. +- `href` *(str)*: The link target +- `text` *(str)*: Link text +- `title` *(str)*: Title attribute +- `context` *(str)*: Surrounding text snippet +- `domain` *(str)*: If external, the domain +```python +for link in result.links["internal"]: + print(f"Internal link to {link['href']} with text {link['text']}") +``` +## 5. Additional Fields +### 5.1 **`extracted_content`** *(Optional[str])* +**What**: If you used **`extraction_strategy`** (CSS, LLM, etc.), the structured output (JSON). +```python +if result.extracted_content: + data = json.loads(result.extracted_content) + print(data) +``` +### 5.2 **`downloaded_files`** *(Optional[List[str]])* +**What**: If `accept_downloads=True` in your `BrowserConfig` + `downloads_path`, lists local file paths for downloaded items. +```python +if result.downloaded_files: + for file_path in result.downloaded_files: + print("Downloaded:", file_path) +``` +### 5.3 **`screenshot`** *(Optional[str])* +**What**: Base64-encoded screenshot if `screenshot=True` in `CrawlerRunConfig`. +```python +import base64 +if result.screenshot: + with open("page.png", "wb") as f: + f.write(base64.b64decode(result.screenshot)) +``` +### 5.4 **`pdf`** *(Optional[bytes])* +**What**: Raw PDF bytes if `pdf=True` in `CrawlerRunConfig`. +```python +if result.pdf: + with open("page.pdf", "wb") as f: + f.write(result.pdf) +``` +### 5.5 **`mhtml`** *(Optional[str])* +**What**: MHTML snapshot of the page if `capture_mhtml=True` in `CrawlerRunConfig`. MHTML (MIME HTML) format preserves the entire web page with all its resources (CSS, images, scripts, etc.) in a single file. +```python +if result.mhtml: + with open("page.mhtml", "w", encoding="utf-8") as f: + f.write(result.mhtml) +``` +### 5.6 **`metadata`** *(Optional[dict])* +```python +if result.metadata: + print("Title:", result.metadata.get("title")) + print("Author:", result.metadata.get("author")) +``` +## 6. `dispatch_result` (optional) +A `DispatchResult` object providing additional concurrency and resource usage information when crawling URLs in parallel (e.g., via `arun_many()` with custom dispatchers). It contains: +- **`task_id`**: A unique identifier for the parallel task. +- **`memory_usage`** (float): The memory (in MB) used at the time of completion. +- **`peak_memory`** (float): The peak memory usage (in MB) recorded during the task's execution. +- **`start_time`** / **`end_time`** (datetime): Time range for this crawling task. +- **`error_message`** (str): Any dispatcher- or concurrency-related error encountered. +```python +# Example usage: +for result in results: + if result.success and result.dispatch_result: + dr = result.dispatch_result + print(f"URL: {result.url}, Task ID: {dr.task_id}") + print(f"Memory: {dr.memory_usage:.1f} MB (Peak: {dr.peak_memory:.1f} MB)") + print(f"Duration: {dr.end_time - dr.start_time}") +``` +> **Note**: This field is typically populated when using `arun_many(...)` alongside a **dispatcher** (e.g., `MemoryAdaptiveDispatcher` or `SemaphoreDispatcher`). If no concurrency or dispatcher is used, `dispatch_result` may remain `None`. +## 7. Network Requests & Console Messages +When you enable network and console message capturing in `CrawlerRunConfig` using `capture_network_requests=True` and `capture_console_messages=True`, the `CrawlResult` will include these fields: +### 7.1 **`network_requests`** *(Optional[List[Dict[str, Any]]])* +- Each item has an `event_type` field that can be `"request"`, `"response"`, or `"request_failed"`. +- Request events include `url`, `method`, `headers`, `post_data`, `resource_type`, and `is_navigation_request`. +- Response events include `url`, `status`, `status_text`, `headers`, and `request_timing`. +- Failed request events include `url`, `method`, `resource_type`, and `failure_text`. +- All events include a `timestamp` field. +```python +if result.network_requests: + # Count different types of events + requests = [r for r in result.network_requests if r.get("event_type") == "request"] + responses = [r for r in result.network_requests if r.get("event_type") == "response"] + failures = [r for r in result.network_requests if r.get("event_type") == "request_failed"] + + print(f"Captured {len(requests)} requests, {len(responses)} responses, and {len(failures)} failures") + + # Analyze API calls + api_calls = [r for r in requests if "api" in r.get("url", "")] + + # Identify failed resources + for failure in failures: + print(f"Failed to load: {failure.get('url')} - {failure.get('failure_text')}") +``` +### 7.2 **`console_messages`** *(Optional[List[Dict[str, Any]]])* +- Each item has a `type` field indicating the message type (e.g., `"log"`, `"error"`, `"warning"`, etc.). +- The `text` field contains the actual message text. +- Some messages include `location` information (URL, line, column). +- All messages include a `timestamp` field. +```python +if result.console_messages: + # Count messages by type + message_types = {} + for msg in result.console_messages: + msg_type = msg.get("type", "unknown") + message_types[msg_type] = message_types.get(msg_type, 0) + 1 + + print(f"Message type counts: {message_types}") + + # Display errors (which are usually most important) + for msg in result.console_messages: + if msg.get("type") == "error": + print(f"Error: {msg.get('text')}") +``` +## 8. Example: Accessing Everything +```python +async def handle_result(result: CrawlResult): + if not result.success: + print("Crawl error:", result.error_message) + return + + # Basic info + print("Crawled URL:", result.url) + print("Status code:", result.status_code) + + # HTML + print("Original HTML size:", len(result.html)) + print("Cleaned HTML size:", len(result.cleaned_html or "")) + + # Markdown output + if result.markdown: + print("Raw Markdown:", result.markdown.raw_markdown[:300]) + print("Citations Markdown:", result.markdown.markdown_with_citations[:300]) + if result.markdown.fit_markdown: + print("Fit Markdown:", result.markdown.fit_markdown[:200]) + + # Media & Links + if "images" in result.media: + print("Image count:", len(result.media["images"])) + if "internal" in result.links: + print("Internal link count:", len(result.links["internal"])) + + # Extraction strategy result + if result.extracted_content: + print("Structured data:", result.extracted_content) + + # Screenshot/PDF/MHTML + if result.screenshot: + print("Screenshot length:", len(result.screenshot)) + if result.pdf: + print("PDF bytes length:", len(result.pdf)) + if result.mhtml: + print("MHTML length:", len(result.mhtml)) + + # Network and console capturing + if result.network_requests: + print(f"Network requests captured: {len(result.network_requests)}") + # Analyze request types + req_types = {} + for req in result.network_requests: + if "resource_type" in req: + req_types[req["resource_type"]] = req_types.get(req["resource_type"], 0) + 1 + print(f"Resource types: {req_types}") + + if result.console_messages: + print(f"Console messages captured: {len(result.console_messages)}") + # Count by message type + msg_types = {} + for msg in result.console_messages: + msg_types[msg.get("type", "unknown")] = msg_types.get(msg.get("type", "unknown"), 0) + 1 + print(f"Message types: {msg_types}") +``` +## 9. Key Points & Future +1. **Deprecated legacy properties of CrawlResult** + - `markdown_v2` - Deprecated in v0.5. Just use `markdown`. It holds the `MarkdownGenerationResult` now! + - `fit_markdown` and `fit_html` - Deprecated in v0.5. They can now be accessed via `MarkdownGenerationResult` in `result.markdown`. eg: `result.markdown.fit_markdown` and `result.markdown.fit_html` +2. **Fit Content** + - **`fit_markdown`** and **`fit_html`** appear in MarkdownGenerationResult, only if you used a content filter (like **PruningContentFilter** or **BM25ContentFilter**) inside your **MarkdownGenerationStrategy** or set them directly. + - If no filter is used, they remain `None`. +3. **References & Citations** + - If you enable link citations in your `DefaultMarkdownGenerator` (`options={"citations": True}`), you’ll see `markdown_with_citations` plus a **`references_markdown`** block. This helps large language models or academic-like referencing. +4. **Links & Media** + - `links["internal"]` and `links["external"]` group discovered anchors by domain. + - `media["images"]` / `["videos"]` / `["audios"]` store extracted media elements with optional scoring or context. +5. **Error Cases** + - If `success=False`, check `error_message` (e.g., timeouts, invalid URLs). + - `status_code` might be `None` if we failed before an HTTP response. +Use **`CrawlResult`** to glean all final outputs and feed them into your data pipelines, AI models, or archives. With the synergy of a properly configured **BrowserConfig** and **CrawlerRunConfig**, the crawler can produce robust, structured results here in **`CrawlResult`**. + + + +# Configuration + +# Browser, Crawler & LLM Configuration (Quick Overview) +Crawl4AI's flexibility stems from two key classes: +1. **`BrowserConfig`** – Dictates **how** the browser is launched and behaves (e.g., headless or visible, proxy, user agent). +2. **`CrawlerRunConfig`** – Dictates **how** each **crawl** operates (e.g., caching, extraction, timeouts, JavaScript code to run, etc.). +3. **`LLMConfig`** - Dictates **how** LLM providers are configured. (model, api token, base url, temperature etc.) +In most examples, you create **one** `BrowserConfig` for the entire crawler session, then pass a **fresh** or re-used `CrawlerRunConfig` whenever you call `arun()`. This tutorial shows the most commonly used parameters. If you need advanced or rarely used fields, see the [Configuration Parameters](../api/parameters.md). +## 1. BrowserConfig Essentials +```python +class BrowserConfig: + def __init__( + browser_type="chromium", + headless=True, + proxy_config=None, + viewport_width=1080, + viewport_height=600, + verbose=True, + use_persistent_context=False, + user_data_dir=None, + cookies=None, + headers=None, + user_agent=None, + text_mode=False, + light_mode=False, + extra_args=None, + enable_stealth=False, + # ... other advanced parameters omitted here + ): + ... +``` +### Key Fields to Note +1. **`browser_type`** +- Options: `"chromium"`, `"firefox"`, or `"webkit"`. +- Defaults to `"chromium"`. +- If you need a different engine, specify it here. +2. **`headless`** + - `True`: Runs the browser in headless mode (invisible browser). + - `False`: Runs the browser in visible mode, which helps with debugging. +3. **`proxy_config`** + - A dictionary with fields like: +```json +{ + "server": "http://proxy.example.com:8080", + "username": "...", + "password": "..." +} +``` + - Leave as `None` if a proxy is not required. +4. **`viewport_width` & `viewport_height`**: + - The initial window size. + - Some sites behave differently with smaller or bigger viewports. +5. **`verbose`**: + - If `True`, prints extra logs. + - Handy for debugging. +6. **`use_persistent_context`**: + - If `True`, uses a **persistent** browser profile, storing cookies/local storage across runs. + - Typically also set `user_data_dir` to point to a folder. +7. **`cookies`** & **`headers`**: + - E.g. `cookies=[{"name": "session", "value": "abc123", "domain": "example.com"}]`. +8. **`user_agent`**: + - Custom User-Agent string. If `None`, a default is used. + - You can also set `user_agent_mode="random"` for randomization (if you want to fight bot detection). +9. **`text_mode`** & **`light_mode`**: + - `text_mode=True` disables images, possibly speeding up text-only crawls. + - `light_mode=True` turns off certain background features for performance. +10. **`extra_args`**: + - Additional flags for the underlying browser. + - E.g. `["--disable-extensions"]`. +11. **`enable_stealth`**: + - If `True`, enables stealth mode using playwright-stealth. + - Modifies browser fingerprints to avoid basic bot detection. + - Default is `False`. Recommended for sites with bot protection. +### Helper Methods +Both configuration classes provide a `clone()` method to create modified copies: +```python +# Create a base browser config +base_browser = BrowserConfig( + browser_type="chromium", + headless=True, + text_mode=True +) + +# Create a visible browser config for debugging +debug_browser = base_browser.clone( + headless=False, + verbose=True +) +``` +```python +from crawl4ai import AsyncWebCrawler, BrowserConfig + +browser_conf = BrowserConfig( + browser_type="firefox", + headless=False, + text_mode=True +) + +async with AsyncWebCrawler(config=browser_conf) as crawler: + result = await crawler.arun("https://example.com") + print(result.markdown[:300]) +``` +## 2. CrawlerRunConfig Essentials +```python +class CrawlerRunConfig: + def __init__( + word_count_threshold=200, + extraction_strategy=None, + markdown_generator=None, + cache_mode=None, + js_code=None, + wait_for=None, + screenshot=False, + pdf=False, + capture_mhtml=False, + # Location and Identity Parameters + locale=None, # e.g. "en-US", "fr-FR" + timezone_id=None, # e.g. "America/New_York" + geolocation=None, # GeolocationConfig object + # Resource Management + enable_rate_limiting=False, + rate_limit_config=None, + memory_threshold_percent=70.0, + check_interval=1.0, + max_session_permit=20, + display_mode=None, + verbose=True, + stream=False, # Enable streaming for arun_many() + # ... other advanced parameters omitted + ): + ... +``` +### Key Fields to Note +1. **`word_count_threshold`**: + - The minimum word count before a block is considered. + - If your site has lots of short paragraphs or items, you can lower it. +2. **`extraction_strategy`**: + - Where you plug in JSON-based extraction (CSS, LLM, etc.). + - If `None`, no structured extraction is done (only raw/cleaned HTML + markdown). +3. **`markdown_generator`**: + - E.g., `DefaultMarkdownGenerator(...)`, controlling how HTML→Markdown conversion is done. + - If `None`, a default approach is used. +4. **`cache_mode`**: + - Controls caching behavior (`ENABLED`, `BYPASS`, `DISABLED`, etc.). + - If `None`, defaults to some level of caching or you can specify `CacheMode.ENABLED`. +5. **`js_code`**: + - A string or list of JS strings to execute. + - Great for "Load More" buttons or user interactions. +6. **`wait_for`**: + - A CSS or JS expression to wait for before extracting content. + - Common usage: `wait_for="css:.main-loaded"` or `wait_for="js:() => window.loaded === true"`. +7. **`screenshot`**, **`pdf`**, & **`capture_mhtml`**: + - If `True`, captures a screenshot, PDF, or MHTML snapshot after the page is fully loaded. + - The results go to `result.screenshot` (base64), `result.pdf` (bytes), or `result.mhtml` (string). +8. **Location Parameters**: + - **`locale`**: Browser's locale (e.g., `"en-US"`, `"fr-FR"`) for language preferences + - **`timezone_id`**: Browser's timezone (e.g., `"America/New_York"`, `"Europe/Paris"`) + - **`geolocation`**: GPS coordinates via `GeolocationConfig(latitude=48.8566, longitude=2.3522)` +9. **`verbose`**: + - Logs additional runtime details. + - Overlaps with the browser's verbosity if also set to `True` in `BrowserConfig`. +10. **`enable_rate_limiting`**: + - If `True`, enables rate limiting for batch processing. + - Requires `rate_limit_config` to be set. +11. **`memory_threshold_percent`**: + - The memory threshold (as a percentage) to monitor. + - If exceeded, the crawler will pause or slow down. +12. **`check_interval`**: + - The interval (in seconds) to check system resources. + - Affects how often memory and CPU usage are monitored. +13. **`max_session_permit`**: + - The maximum number of concurrent crawl sessions. + - Helps prevent overwhelming the system. +14. **`url_matcher`** & **`match_mode`**: + - Enable URL-specific configurations when used with `arun_many()`. + - Set `url_matcher` to patterns (glob, function, or list) to match specific URLs. + - Use `match_mode` (OR/AND) to control how multiple patterns combine. +15. **`display_mode`**: + - The display mode for progress information (`DETAILED`, `BRIEF`, etc.). + - Affects how much information is printed during the crawl. +### Helper Methods +The `clone()` method is particularly useful for creating variations of your crawler configuration: +```python +# Create a base configuration +base_config = CrawlerRunConfig( + cache_mode=CacheMode.ENABLED, + word_count_threshold=200, + wait_until="networkidle" +) + +# Create variations for different use cases +stream_config = base_config.clone( + stream=True, # Enable streaming mode + cache_mode=CacheMode.BYPASS +) + +debug_config = base_config.clone( + page_timeout=120000, # Longer timeout for debugging + verbose=True +) +``` +The `clone()` method: +- Creates a new instance with all the same settings +- Updates only the specified parameters +- Leaves the original configuration unchanged +- Perfect for creating variations without repeating all parameters +## 3. LLMConfig Essentials +### Key fields to note +1. **`provider`**: +- Which LLM provider to use. +- Possible values are `"ollama/llama3","groq/llama3-70b-8192","groq/llama3-8b-8192", "openai/gpt-4o-mini" ,"openai/gpt-4o","openai/o1-mini","openai/o1-preview","openai/o3-mini","openai/o3-mini-high","anthropic/claude-3-haiku-20240307","anthropic/claude-3-opus-20240229","anthropic/claude-3-sonnet-20240229","anthropic/claude-3-5-sonnet-20240620","gemini/gemini-pro","gemini/gemini-1.5-pro","gemini/gemini-2.0-flash","gemini/gemini-2.0-flash-exp","gemini/gemini-2.0-flash-lite-preview-02-05","deepseek/deepseek-chat"`
*(default: `"openai/gpt-4o-mini"`)* +2. **`api_token`**: + - Optional. When not provided explicitly, api_token will be read from environment variables based on provider. For example: If a gemini model is passed as provider then,`"GEMINI_API_KEY"` will be read from environment variables + - API token of LLM provider
eg: `api_token = "gsk_1ClHGGJ7Lpn4WGybR7vNWGdyb3FY7zXEw3SCiy0BAVM9lL8CQv"` + - Environment variable - use with prefix "env:"
eg:`api_token = "env: GROQ_API_KEY"` +3. **`base_url`**: + - If your provider has a custom endpoint +```python +llm_config = LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv("OPENAI_API_KEY")) +``` +## 4. Putting It All Together +In a typical scenario, you define **one** `BrowserConfig` for your crawler session, then create **one or more** `CrawlerRunConfig` & `LLMConfig` depending on each call's needs: +```python +import asyncio +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LLMConfig, LLMContentFilter, DefaultMarkdownGenerator +from crawl4ai import JsonCssExtractionStrategy + +async def main(): + # 1) Browser config: headless, bigger viewport, no proxy + browser_conf = BrowserConfig( + headless=True, + viewport_width=1280, + viewport_height=720 + ) + + # 2) Example extraction strategy + schema = { + "name": "Articles", + "baseSelector": "div.article", + "fields": [ + {"name": "title", "selector": "h2", "type": "text"}, + {"name": "link", "selector": "a", "type": "attribute", "attribute": "href"} + ] + } + extraction = JsonCssExtractionStrategy(schema) + + # 3) Example LLM content filtering + + gemini_config = LLMConfig( + provider="gemini/gemini-1.5-pro", + api_token = "env:GEMINI_API_TOKEN" + ) + + # Initialize LLM filter with specific instruction + filter = LLMContentFilter( + llm_config=gemini_config, # or your preferred provider + instruction=""" + Focus on extracting the core educational content. + Include: + - Key concepts and explanations + - Important code examples + - Essential technical details + Exclude: + - Navigation elements + - Sidebars + - Footer content + Format the output as clean markdown with proper code blocks and headers. + """, + chunk_token_threshold=500, # Adjust based on your needs + verbose=True + ) + + md_generator = DefaultMarkdownGenerator( + content_filter=filter, + options={"ignore_links": True} + ) + + # 4) Crawler run config: skip cache, use extraction + run_conf = CrawlerRunConfig( + markdown_generator=md_generator, + extraction_strategy=extraction, + cache_mode=CacheMode.BYPASS, + ) + + async with AsyncWebCrawler(config=browser_conf) as crawler: + # 4) Execute the crawl + result = await crawler.arun(url="https://example.com/news", config=run_conf) + + if result.success: + print("Extracted content:", result.extracted_content) + else: + print("Error:", result.error_message) + +if __name__ == "__main__": + asyncio.run(main()) +``` +## 5. Next Steps +- [BrowserConfig, CrawlerRunConfig & LLMConfig Reference](../api/parameters.md) +- **Custom Hooks & Auth** (Inject JavaScript or handle login forms). +- **Session Management** (Re-use pages, preserve state across multiple calls). +- **Advanced Caching** (Fine-tune read/write cache modes). +## 6. Conclusion + + +# 1. **BrowserConfig** – Controlling the Browser +`BrowserConfig` focuses on **how** the browser is launched and behaves. This includes headless mode, proxies, user agents, and other environment tweaks. +```python +from crawl4ai import AsyncWebCrawler, BrowserConfig + +browser_cfg = BrowserConfig( + browser_type="chromium", + headless=True, + viewport_width=1280, + viewport_height=720, + proxy="http://user:pass@proxy:8080", + user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/116.0.0.0 Safari/537.36", +) +``` +## 1.1 Parameter Highlights +| **Parameter** | **Type / Default** | **What It Does** | +|-----------------------|----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------| +| **`browser_type`** | `"chromium"`, `"firefox"`, `"webkit"`
*(default: `"chromium"`)* | Which browser engine to use. `"chromium"` is typical for many sites, `"firefox"` or `"webkit"` for specialized tests. | +| **`headless`** | `bool` (default: `True`) | Headless means no visible UI. `False` is handy for debugging. | +| **`viewport_width`** | `int` (default: `1080`) | Initial page width (in px). Useful for testing responsive layouts. | +| **`viewport_height`** | `int` (default: `600`) | Initial page height (in px). | +| **`proxy`** | `str` (deprecated) | Deprecated. Use `proxy_config` instead. If set, it will be auto-converted internally. | +| **`proxy_config`** | `dict` (default: `None`) | For advanced or multi-proxy needs, specify details like `{"server": "...", "username": "...", ...}`. | +| **`use_persistent_context`** | `bool` (default: `False`) | If `True`, uses a **persistent** browser context (keep cookies, sessions across runs). Also sets `use_managed_browser=True`. | +| **`user_data_dir`** | `str or None` (default: `None`) | Directory to store user data (profiles, cookies). Must be set if you want permanent sessions. | +| **`ignore_https_errors`** | `bool` (default: `True`) | If `True`, continues despite invalid certificates (common in dev/staging). | +| **`java_script_enabled`** | `bool` (default: `True`) | Disable if you want no JS overhead, or if only static content is needed. | +| **`cookies`** | `list` (default: `[]`) | Pre-set cookies, each a dict like `{"name": "session", "value": "...", "url": "..."}`. | +| **`headers`** | `dict` (default: `{}`) | Extra HTTP headers for every request, e.g. `{"Accept-Language": "en-US"}`. | +| **`user_agent`** | `str` (default: Chrome-based UA) | Your custom or random user agent. `user_agent_mode="random"` can shuffle it. | +| **`light_mode`** | `bool` (default: `False`) | Disables some background features for performance gains. | +| **`text_mode`** | `bool` (default: `False`) | If `True`, tries to disable images/other heavy content for speed. | +| **`use_managed_browser`** | `bool` (default: `False`) | For advanced “managed” interactions (debugging, CDP usage). Typically set automatically if persistent context is on. | +| **`extra_args`** | `list` (default: `[]`) | Additional flags for the underlying browser process, e.g. `["--disable-extensions"]`. | +- Set `headless=False` to visually **debug** how pages load or how interactions proceed. +- If you need **authentication** storage or repeated sessions, consider `use_persistent_context=True` and specify `user_data_dir`. +- For large pages, you might need a bigger `viewport_width` and `viewport_height` to handle dynamic content. +# 2. **CrawlerRunConfig** – Controlling Each Crawl +While `BrowserConfig` sets up the **environment**, `CrawlerRunConfig` details **how** each **crawl operation** should behave: caching, content filtering, link or domain blocking, timeouts, JavaScript code, etc. +```python +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig + +run_cfg = CrawlerRunConfig( + wait_for="css:.main-content", + word_count_threshold=15, + excluded_tags=["nav", "footer"], + exclude_external_links=True, + stream=True, # Enable streaming for arun_many() +) +``` +## 2.1 Parameter Highlights +### A) **Content Processing** +| **Parameter** | **Type / Default** | **What It Does** | +|------------------------------|--------------------------------------|-------------------------------------------------------------------------------------------------| +| **`word_count_threshold`** | `int` (default: ~200) | Skips text blocks below X words. Helps ignore trivial sections. | +| **`extraction_strategy`** | `ExtractionStrategy` (default: None) | If set, extracts structured data (CSS-based, LLM-based, etc.). | +| **`markdown_generator`** | `MarkdownGenerationStrategy` (None) | If you want specialized markdown output (citations, filtering, chunking, etc.). Can be customized with options such as `content_source` parameter to select the HTML input source ('cleaned_html', 'raw_html', or 'fit_html'). | +| **`css_selector`** | `str` (None) | Retains only the part of the page matching this selector. Affects the entire extraction process. | +| **`target_elements`** | `List[str]` (None) | List of CSS selectors for elements to focus on for markdown generation and data extraction, while still processing the entire page for links, media, etc. Provides more flexibility than `css_selector`. | +| **`excluded_tags`** | `list` (None) | Removes entire tags (e.g. `["script", "style"]`). | +| **`excluded_selector`** | `str` (None) | Like `css_selector` but to exclude. E.g. `"#ads, .tracker"`. | +| **`only_text`** | `bool` (False) | If `True`, tries to extract text-only content. | +| **`prettiify`** | `bool` (False) | If `True`, beautifies final HTML (slower, purely cosmetic). | +| **`keep_data_attributes`** | `bool` (False) | If `True`, preserve `data-*` attributes in cleaned HTML. | +| **`remove_forms`** | `bool` (False) | If `True`, remove all `` elements. | +### B) **Caching & Session** +| **Parameter** | **Type / Default** | **What It Does** | +|-------------------------|------------------------|------------------------------------------------------------------------------------------------------------------------------| +| **`cache_mode`** | `CacheMode or None` | Controls how caching is handled (`ENABLED`, `BYPASS`, `DISABLED`, etc.). If `None`, typically defaults to `ENABLED`. | +| **`session_id`** | `str or None` | Assign a unique ID to reuse a single browser session across multiple `arun()` calls. | +| **`bypass_cache`** | `bool` (False) | If `True`, acts like `CacheMode.BYPASS`. | +| **`disable_cache`** | `bool` (False) | If `True`, acts like `CacheMode.DISABLED`. | +| **`no_cache_read`** | `bool` (False) | If `True`, acts like `CacheMode.WRITE_ONLY` (writes cache but never reads). | +| **`no_cache_write`** | `bool` (False) | If `True`, acts like `CacheMode.READ_ONLY` (reads cache but never writes). | +### C) **Page Navigation & Timing** +| **Parameter** | **Type / Default** | **What It Does** | +|----------------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------| +| **`wait_until`** | `str` (domcontentloaded)| Condition for navigation to “complete”. Often `"networkidle"` or `"domcontentloaded"`. | +| **`page_timeout`** | `int` (60000 ms) | Timeout for page navigation or JS steps. Increase for slow sites. | +| **`wait_for`** | `str or None` | Wait for a CSS (`"css:selector"`) or JS (`"js:() => bool"`) condition before content extraction. | +| **`wait_for_images`** | `bool` (False) | Wait for images to load before finishing. Slows down if you only want text. | +| **`delay_before_return_html`** | `float` (0.1) | Additional pause (seconds) before final HTML is captured. Good for last-second updates. | +| **`check_robots_txt`** | `bool` (False) | Whether to check and respect robots.txt rules before crawling. If True, caches robots.txt for efficiency. | +| **`mean_delay`** and **`max_range`** | `float` (0.1, 0.3) | If you call `arun_many()`, these define random delay intervals between crawls, helping avoid detection or rate limits. | +| **`semaphore_count`** | `int` (5) | Max concurrency for `arun_many()`. Increase if you have resources for parallel crawls. | +### D) **Page Interaction** +| **Parameter** | **Type / Default** | **What It Does** | +|----------------------------|--------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------| +| **`js_code`** | `str or list[str]` (None) | JavaScript to run after load. E.g. `"document.querySelector('button')?.click();"`. | +| **`js_only`** | `bool` (False) | If `True`, indicates we’re reusing an existing session and only applying JS. No full reload. | +| **`ignore_body_visibility`** | `bool` (True) | Skip checking if `` is visible. Usually best to keep `True`. | +| **`scan_full_page`** | `bool` (False) | If `True`, auto-scroll the page to load dynamic content (infinite scroll). | +| **`scroll_delay`** | `float` (0.2) | Delay between scroll steps if `scan_full_page=True`. | +| **`process_iframes`** | `bool` (False) | Inlines iframe content for single-page extraction. | +| **`remove_overlay_elements`** | `bool` (False) | Removes potential modals/popups blocking the main content. | +| **`simulate_user`** | `bool` (False) | Simulate user interactions (mouse movements) to avoid bot detection. | +| **`override_navigator`** | `bool` (False) | Override `navigator` properties in JS for stealth. | +| **`magic`** | `bool` (False) | Automatic handling of popups/consent banners. Experimental. | +| **`adjust_viewport_to_content`** | `bool` (False) | Resizes viewport to match page content height. | +If your page is a single-page app with repeated JS updates, set `js_only=True` in subsequent calls, plus a `session_id` for reusing the same tab. +### E) **Media Handling** +| **Parameter** | **Type / Default** | **What It Does** | +|--------------------------------------------|---------------------|-----------------------------------------------------------------------------------------------------------| +| **`screenshot`** | `bool` (False) | Capture a screenshot (base64) in `result.screenshot`. | +| **`screenshot_wait_for`** | `float or None` | Extra wait time before the screenshot. | +| **`screenshot_height_threshold`** | `int` (~20000) | If the page is taller than this, alternate screenshot strategies are used. | +| **`pdf`** | `bool` (False) | If `True`, returns a PDF in `result.pdf`. | +| **`capture_mhtml`** | `bool` (False) | If `True`, captures an MHTML snapshot of the page in `result.mhtml`. MHTML includes all page resources (CSS, images, etc.) in a single file. | +| **`image_description_min_word_threshold`** | `int` (~50) | Minimum words for an image’s alt text or description to be considered valid. | +| **`image_score_threshold`** | `int` (~3) | Filter out low-scoring images. The crawler scores images by relevance (size, context, etc.). | +| **`exclude_external_images`** | `bool` (False) | Exclude images from other domains. | +### F) **Link/Domain Handling** +| **Parameter** | **Type / Default** | **What It Does** | +|------------------------------|-------------------------|-----------------------------------------------------------------------------------------------------------------------------| +| **`exclude_social_media_domains`** | `list` (e.g. Facebook/Twitter) | A default list can be extended. Any link to these domains is removed from final output. | +| **`exclude_external_links`** | `bool` (False) | Removes all links pointing outside the current domain. | +| **`exclude_social_media_links`** | `bool` (False) | Strips links specifically to social sites (like Facebook or Twitter). | +| **`exclude_domains`** | `list` ([]) | Provide a custom list of domains to exclude (like `["ads.com", "trackers.io"]`). | +| **`preserve_https_for_internal_links`** | `bool` (False) | If `True`, preserves HTTPS scheme for internal links even when the server redirects to HTTP. Useful for security-conscious crawling. | +### G) **Debug & Logging** +| **Parameter** | **Type / Default** | **What It Does** | +|----------------|--------------------|---------------------------------------------------------------------------| +| **`verbose`** | `bool` (True) | Prints logs detailing each step of crawling, interactions, or errors. | +| **`log_console`** | `bool` (False) | Logs the page’s JavaScript console output if you want deeper JS debugging.| +### H) **Virtual Scroll Configuration** +| **Parameter** | **Type / Default** | **What It Does** | +|------------------------------|------------------------------|-------------------------------------------------------------------------------------------------------------------------------------| +| **`virtual_scroll_config`** | `VirtualScrollConfig or dict` (None) | Configuration for handling virtualized scrolling on sites like Twitter/Instagram where content is replaced rather than appended. | +When sites use virtual scrolling (content replaced as you scroll), use `VirtualScrollConfig`: +```python +from crawl4ai import VirtualScrollConfig + +virtual_config = VirtualScrollConfig( + container_selector="#timeline", # CSS selector for scrollable container + scroll_count=30, # Number of times to scroll + scroll_by="container_height", # How much to scroll: "container_height", "page_height", or pixels (e.g. 500) + wait_after_scroll=0.5 # Seconds to wait after each scroll for content to load +) + +config = CrawlerRunConfig( + virtual_scroll_config=virtual_config +) +``` +**VirtualScrollConfig Parameters:** +| **Parameter** | **Type / Default** | **What It Does** | +|------------------------|---------------------------|-------------------------------------------------------------------------------------------| +| **`container_selector`** | `str` (required) | CSS selector for the scrollable container (e.g., `"#feed"`, `".timeline"`) | +| **`scroll_count`** | `int` (10) | Maximum number of scrolls to perform | +| **`scroll_by`** | `str or int` ("container_height") | Scroll amount: `"container_height"`, `"page_height"`, or pixels (e.g., `500`) | +| **`wait_after_scroll`** | `float` (0.5) | Time in seconds to wait after each scroll for new content to load | +- Use `virtual_scroll_config` when content is **replaced** during scroll (Twitter, Instagram) +- Use `scan_full_page` when content is **appended** during scroll (traditional infinite scroll) +### I) **URL Matching Configuration** +| **Parameter** | **Type / Default** | **What It Does** | +|------------------------|------------------------------|-------------------------------------------------------------------------------------------------------------------------------------| +| **`url_matcher`** | `UrlMatcher` (None) | Pattern(s) to match URLs against. Can be: string (glob), function, or list of mixed types. **None means match ALL URLs** | +| **`match_mode`** | `MatchMode` (MatchMode.OR) | How to combine multiple matchers in a list: `MatchMode.OR` (any match) or `MatchMode.AND` (all must match) | +The `url_matcher` parameter enables URL-specific configurations when used with `arun_many()`: +```python +from crawl4ai import CrawlerRunConfig, MatchMode +from crawl4ai.processors.pdf import PDFContentScrapingStrategy +from crawl4ai.extraction_strategy import JsonCssExtractionStrategy + +# Simple string pattern (glob-style) +pdf_config = CrawlerRunConfig( + url_matcher="*.pdf", + scraping_strategy=PDFContentScrapingStrategy() +) + +# Multiple patterns with OR logic (default) +blog_config = CrawlerRunConfig( + url_matcher=["*/blog/*", "*/article/*", "*/news/*"], + match_mode=MatchMode.OR # Any pattern matches +) + +# Function matcher +api_config = CrawlerRunConfig( + url_matcher=lambda url: 'api' in url or url.endswith('.json'), + # Other settings like extraction_strategy +) + +# Mixed: String + Function with AND logic +complex_config = CrawlerRunConfig( + url_matcher=[ + lambda url: url.startswith('https://'), # Must be HTTPS + "*.org/*", # Must be .org domain + lambda url: 'docs' in url # Must contain 'docs' + ], + match_mode=MatchMode.AND # ALL conditions must match +) + +# Combined patterns and functions with AND logic +secure_docs = CrawlerRunConfig( + url_matcher=["https://*", lambda url: '.doc' in url], + match_mode=MatchMode.AND # Must be HTTPS AND contain .doc +) + +# Default config - matches ALL URLs +default_config = CrawlerRunConfig() # No url_matcher = matches everything +``` +**UrlMatcher Types:** +- **None (default)**: When `url_matcher` is None or not set, the config matches ALL URLs +- **String patterns**: Glob-style patterns like `"*.pdf"`, `"*/api/*"`, `"https://*.example.com/*"` +- **Functions**: `lambda url: bool` - Custom logic for complex matching +- **Lists**: Mix strings and functions, combined with `MatchMode.OR` or `MatchMode.AND` +**Important Behavior:** +- When passing a list of configs to `arun_many()`, URLs are matched against each config's `url_matcher` in order. First match wins! +- If no config matches a URL and there's no default config (one without `url_matcher`), the URL will fail with "No matching configuration found" +Both `BrowserConfig` and `CrawlerRunConfig` provide a `clone()` method to create modified copies: +```python +# Create a base configuration +base_config = CrawlerRunConfig( + cache_mode=CacheMode.ENABLED, + word_count_threshold=200 +) + +# Create variations using clone() +stream_config = base_config.clone(stream=True) +no_cache_config = base_config.clone( + cache_mode=CacheMode.BYPASS, + stream=True +) +``` +The `clone()` method is particularly useful when you need slightly different configurations for different use cases, without modifying the original config. +## 2.3 Example Usage +```python +import asyncio +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode + +async def main(): + # Configure the browser + browser_cfg = BrowserConfig( + headless=False, + viewport_width=1280, + viewport_height=720, + proxy="http://user:pass@myproxy:8080", + text_mode=True + ) + + # Configure the run + run_cfg = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + session_id="my_session", + css_selector="main.article", + excluded_tags=["script", "style"], + exclude_external_links=True, + wait_for="css:.article-loaded", + screenshot=True, + stream=True + ) + + async with AsyncWebCrawler(config=browser_cfg) as crawler: + result = await crawler.arun( + url="https://example.com/news", + config=run_cfg + ) + if result.success: + print("Final cleaned_html length:", len(result.cleaned_html)) + if result.screenshot: + print("Screenshot captured (base64, length):", len(result.screenshot)) + else: + print("Crawl failed:", result.error_message) + +if __name__ == "__main__": + asyncio.run(main()) +``` +## 2.4 Compliance & Ethics +| **Parameter** | **Type / Default** | **What It Does** | +|-----------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------| +| **`check_robots_txt`**| `bool` (False) | When True, checks and respects robots.txt rules before crawling. Uses efficient caching with SQLite backend. | +| **`user_agent`** | `str` (None) | User agent string to identify your crawler. Used for robots.txt checking when enabled. | +```python +run_config = CrawlerRunConfig( + check_robots_txt=True, # Enable robots.txt compliance + user_agent="MyBot/1.0" # Identify your crawler +) +``` +# 3. **LLMConfig** - Setting up LLM providers +1. LLMExtractionStrategy +2. LLMContentFilter +3. JsonCssExtractionStrategy.generate_schema +4. JsonXPathExtractionStrategy.generate_schema +## 3.1 Parameters +| **Parameter** | **Type / Default** | **What It Does** | +|-----------------------|----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------| +| **`provider`** | `"ollama/llama3","groq/llama3-70b-8192","groq/llama3-8b-8192", "openai/gpt-4o-mini" ,"openai/gpt-4o","openai/o1-mini","openai/o1-preview","openai/o3-mini","openai/o3-mini-high","anthropic/claude-3-haiku-20240307","anthropic/claude-3-opus-20240229","anthropic/claude-3-sonnet-20240229","anthropic/claude-3-5-sonnet-20240620","gemini/gemini-pro","gemini/gemini-1.5-pro","gemini/gemini-2.0-flash","gemini/gemini-2.0-flash-exp","gemini/gemini-2.0-flash-lite-preview-02-05","deepseek/deepseek-chat"`
*(default: `"openai/gpt-4o-mini"`)* | Which LLM provider to use. +| **`api_token`** |1.Optional. When not provided explicitly, api_token will be read from environment variables based on provider. For example: If a gemini model is passed as provider then,`"GEMINI_API_KEY"` will be read from environment variables
2. API token of LLM provider
eg: `api_token = "gsk_1ClHGGJ7Lpn4WGybR7vNWGdyb3FY7zXEw3SCiy0BAVM9lL8CQv"`
3. Environment variable - use with prefix "env:"
eg:`api_token = "env: GROQ_API_KEY"` | API token to use for the given provider +| **`base_url`** |Optional. Custom API endpoint | If your provider has a custom endpoint +## 3.2 Example Usage +```python +llm_config = LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv("OPENAI_API_KEY")) +``` +## 4. Putting It All Together +- **Use** `BrowserConfig` for **global** browser settings: engine, headless, proxy, user agent. +- **Use** `CrawlerRunConfig` for each crawl’s **context**: how to filter content, handle caching, wait for dynamic elements, or run JS. +- **Pass** both configs to `AsyncWebCrawler` (the `BrowserConfig`) and then to `arun()` (the `CrawlerRunConfig`). +- **Use** `LLMConfig` for LLM provider configurations that can be used across all extraction, filtering, schema generation tasks. Can be used in - `LLMExtractionStrategy`, `LLMContentFilter`, `JsonCssExtractionStrategy.generate_schema` & `JsonXPathExtractionStrategy.generate_schema` +```python +# Create a modified copy with the clone() method +stream_cfg = run_cfg.clone( + stream=True, + cache_mode=CacheMode.BYPASS +) +``` + + + +# Crawling Patterns + +# Simple Crawling +## Basic Usage +Set up a simple crawl using `BrowserConfig` and `CrawlerRunConfig`: +```python +import asyncio +from crawl4ai import AsyncWebCrawler +from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig + +async def main(): + browser_config = BrowserConfig() # Default browser configuration + run_config = CrawlerRunConfig() # Default crawl run configuration + + async with AsyncWebCrawler(config=browser_config) as crawler: + result = await crawler.arun( + url="https://example.com", + config=run_config + ) + print(result.markdown) # Print clean markdown content + +if __name__ == "__main__": + asyncio.run(main()) +``` +## Understanding the Response +The `arun()` method returns a `CrawlResult` object with several useful properties. Here's a quick overview (see [CrawlResult](../api/crawl-result.md) for complete details): +```python +config = CrawlerRunConfig( + markdown_generator=DefaultMarkdownGenerator( + content_filter=PruningContentFilter(threshold=0.6), + options={"ignore_links": True} + ) +) + +result = await crawler.arun( + url="https://example.com", + config=config +) + +# Different content formats +print(result.html) # Raw HTML +print(result.cleaned_html) # Cleaned HTML +print(result.markdown.raw_markdown) # Raw markdown from cleaned html +print(result.markdown.fit_markdown) # Most relevant content in markdown + +# Check success status +print(result.success) # True if crawl succeeded +print(result.status_code) # HTTP status code (e.g., 200, 404) + +# Access extracted media and links +print(result.media) # Dictionary of found media (images, videos, audio) +print(result.links) # Dictionary of internal and external links +``` +## Adding Basic Options +Customize your crawl using `CrawlerRunConfig`: +```python +run_config = CrawlerRunConfig( + word_count_threshold=10, # Minimum words per content block + exclude_external_links=True, # Remove external links + remove_overlay_elements=True, # Remove popups/modals + process_iframes=True # Process iframe content +) + +result = await crawler.arun( + url="https://example.com", + config=run_config +) +``` +## Handling Errors +```python +run_config = CrawlerRunConfig() +result = await crawler.arun(url="https://example.com", config=run_config) + +if not result.success: + print(f"Crawl failed: {result.error_message}") + print(f"Status code: {result.status_code}") +``` +## Logging and Debugging +Enable verbose logging in `BrowserConfig`: +```python +browser_config = BrowserConfig(verbose=True) + +async with AsyncWebCrawler(config=browser_config) as crawler: + run_config = CrawlerRunConfig() + result = await crawler.arun(url="https://example.com", config=run_config) +``` +## Complete Example +```python +import asyncio +from crawl4ai import AsyncWebCrawler +from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig, CacheMode + +async def main(): + browser_config = BrowserConfig(verbose=True) + run_config = CrawlerRunConfig( + # Content filtering + word_count_threshold=10, + excluded_tags=['form', 'header'], + exclude_external_links=True, + + # Content processing + process_iframes=True, + remove_overlay_elements=True, + + # Cache control + cache_mode=CacheMode.ENABLED # Use cache if available + ) + + async with AsyncWebCrawler(config=browser_config) as crawler: + result = await crawler.arun( + url="https://example.com", + config=run_config + ) + + if result.success: + # Print clean content + print("Content:", result.markdown[:500]) # First 500 chars + + # Process images + for image in result.media["images"]: + print(f"Found image: {image['src']}") + + # Process links + for link in result.links["internal"]: + print(f"Internal link: {link['href']}") + + else: + print(f"Crawl failed: {result.error_message}") + +if __name__ == "__main__": + asyncio.run(main()) +``` + + + +# Content Processing + +# Markdown Generation Basics +1. How to configure the **Default Markdown Generator** +3. The difference between raw markdown (`result.markdown`) and filtered markdown (`fit_markdown`) +> - You know how to configure `CrawlerRunConfig`. +## 1. Quick Example +```python +import asyncio +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig +from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator + +async def main(): + config = CrawlerRunConfig( + markdown_generator=DefaultMarkdownGenerator() + ) + async with AsyncWebCrawler() as crawler: + result = await crawler.arun("https://example.com", config=config) + + if result.success: + print("Raw Markdown Output:\n") + print(result.markdown) # The unfiltered markdown from the page + else: + print("Crawl failed:", result.error_message) + +if __name__ == "__main__": + asyncio.run(main()) +``` +- `CrawlerRunConfig( markdown_generator = DefaultMarkdownGenerator() )` instructs Crawl4AI to convert the final HTML into markdown at the end of each crawl. +- The resulting markdown is accessible via `result.markdown`. +## 2. How Markdown Generation Works +### 2.1 HTML-to-Text Conversion (Forked & Modified) +- Preserves headings, code blocks, bullet points, etc. +- Removes extraneous tags (scripts, styles) that don’t add meaningful content. +- Can optionally generate references for links or skip them altogether. +### 2.2 Link Citations & References +By default, the generator can convert `` elements into `[text][1]` citations, then place the actual links at the bottom of the document. This is handy for research workflows that demand references in a structured manner. +### 2.3 Optional Content Filters +## 3. Configuring the Default Markdown Generator +You can tweak the output by passing an `options` dict to `DefaultMarkdownGenerator`. For example: +```python +from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig + +async def main(): + # Example: ignore all links, don't escape HTML, and wrap text at 80 characters + md_generator = DefaultMarkdownGenerator( + options={ + "ignore_links": True, + "escape_html": False, + "body_width": 80 + } + ) + + config = CrawlerRunConfig( + markdown_generator=md_generator + ) + + async with AsyncWebCrawler() as crawler: + result = await crawler.arun("https://example.com/docs", config=config) + if result.success: + print("Markdown:\n", result.markdown[:500]) # Just a snippet + else: + print("Crawl failed:", result.error_message) + +if __name__ == "__main__": + import asyncio + asyncio.run(main()) +``` +Some commonly used `options`: +- **`ignore_links`** (bool): Whether to remove all hyperlinks in the final markdown. +- **`ignore_images`** (bool): Remove all `![image]()` references. +- **`escape_html`** (bool): Turn HTML entities into text (default is often `True`). +- **`body_width`** (int): Wrap text at N characters. `0` or `None` means no wrapping. +- **`skip_internal_links`** (bool): If `True`, omit `#localAnchors` or internal links referencing the same page. +- **`include_sup_sub`** (bool): Attempt to handle `` / `` in a more readable way. +## 4. Selecting the HTML Source for Markdown Generation +The `content_source` parameter allows you to control which HTML content is used as input for markdown generation. This gives you flexibility in how the HTML is processed before conversion to markdown. +```python +from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig + +async def main(): + # Option 1: Use the raw HTML directly from the webpage (before any processing) + raw_md_generator = DefaultMarkdownGenerator( + content_source="raw_html", + options={"ignore_links": True} + ) + + # Option 2: Use the cleaned HTML (after scraping strategy processing - default) + cleaned_md_generator = DefaultMarkdownGenerator( + content_source="cleaned_html", # This is the default + options={"ignore_links": True} + ) + + # Option 3: Use preprocessed HTML optimized for schema extraction + fit_md_generator = DefaultMarkdownGenerator( + content_source="fit_html", + options={"ignore_links": True} + ) + + # Use one of the generators in your crawler config + config = CrawlerRunConfig( + markdown_generator=raw_md_generator # Try each of the generators + ) + + async with AsyncWebCrawler() as crawler: + result = await crawler.arun("https://example.com", config=config) + if result.success: + print("Markdown:\n", result.markdown.raw_markdown[:500]) + else: + print("Crawl failed:", result.error_message) + +if __name__ == "__main__": + import asyncio + asyncio.run(main()) +``` +### HTML Source Options +- **`"cleaned_html"`** (default): Uses the HTML after it has been processed by the scraping strategy. This HTML is typically cleaner and more focused on content, with some boilerplate removed. +- **`"raw_html"`**: Uses the original HTML directly from the webpage, before any cleaning or processing. This preserves more of the original content, but may include navigation bars, ads, footers, and other elements that might not be relevant to the main content. +- **`"fit_html"`**: Uses HTML preprocessed for schema extraction. This HTML is optimized for structured data extraction and may have certain elements simplified or removed. +### When to Use Each Option +- Use **`"cleaned_html"`** (default) for most cases where you want a balance of content preservation and noise removal. +- Use **`"raw_html"`** when you need to preserve all original content, or when the cleaning process is removing content you actually want to keep. +- Use **`"fit_html"`** when working with structured data or when you need HTML that's optimized for schema extraction. +## 5. Content Filters +### 5.1 BM25ContentFilter +```python +from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator +from crawl4ai.content_filter_strategy import BM25ContentFilter +from crawl4ai import CrawlerRunConfig + +bm25_filter = BM25ContentFilter( + user_query="machine learning", + bm25_threshold=1.2, + language="english" +) + +md_generator = DefaultMarkdownGenerator( + content_filter=bm25_filter, + options={"ignore_links": True} +) + +config = CrawlerRunConfig(markdown_generator=md_generator) +``` +- **`user_query`**: The term you want to focus on. BM25 tries to keep only content blocks relevant to that query. +- **`bm25_threshold`**: Raise it to keep fewer blocks; lower it to keep more. +- **`use_stemming`** *(default `True`)*: Whether to apply stemming to the query and content. +- **`language (str)`**: Language for stemming (default: 'english'). +### 5.2 PruningContentFilter +If you **don’t** have a specific query, or if you just want a robust “junk remover,” use `PruningContentFilter`. It analyzes text density, link density, HTML structure, and known patterns (like “nav,” “footer”) to systematically prune extraneous or repetitive sections. +```python +from crawl4ai.content_filter_strategy import PruningContentFilter + +prune_filter = PruningContentFilter( + threshold=0.5, + threshold_type="fixed", # or "dynamic" + min_word_threshold=50 +) +``` +- **`threshold`**: Score boundary. Blocks below this score get removed. +- **`threshold_type`**: + - `"fixed"`: Straight comparison (`score >= threshold` keeps the block). + - `"dynamic"`: The filter adjusts threshold in a data-driven manner. +- **`min_word_threshold`**: Discard blocks under N words as likely too short or unhelpful. +- You want a broad cleanup without a user query. +### 5.3 LLMContentFilter +```python +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, LLMConfig, DefaultMarkdownGenerator +from crawl4ai.content_filter_strategy import LLMContentFilter + +async def main(): + # Initialize LLM filter with specific instruction + filter = LLMContentFilter( + llm_config = LLMConfig(provider="openai/gpt-4o",api_token="your-api-token"), #or use environment variable + instruction=""" + Focus on extracting the core educational content. + Include: + - Key concepts and explanations + - Important code examples + - Essential technical details + Exclude: + - Navigation elements + - Sidebars + - Footer content + Format the output as clean markdown with proper code blocks and headers. + """, + chunk_token_threshold=4096, # Adjust based on your needs + verbose=True + ) + md_generator = DefaultMarkdownGenerator( + content_filter=filter, + options={"ignore_links": True} + ) + config = CrawlerRunConfig( + markdown_generator=md_generator, + ) + + async with AsyncWebCrawler() as crawler: + result = await crawler.arun("https://example.com", config=config) + print(result.markdown.fit_markdown) # Filtered markdown content +``` +- **Chunk Processing**: Handles large documents by processing them in chunks (controlled by `chunk_token_threshold`) +- **Parallel Processing**: For better performance, use smaller `chunk_token_threshold` (e.g., 2048 or 4096) to enable parallel processing of content chunks +1. **Exact Content Preservation**: +```python +filter = LLMContentFilter( + instruction=""" + Extract the main educational content while preserving its original wording and substance completely. + 1. Maintain the exact language and terminology + 2. Keep all technical explanations and examples intact + 3. Preserve the original flow and structure + 4. Remove only clearly irrelevant elements like navigation menus and ads + """, + chunk_token_threshold=4096 +) +``` +2. **Focused Content Extraction**: +```python +filter = LLMContentFilter( + instruction=""" + Focus on extracting specific types of content: + - Technical documentation + - Code examples + - API references + Reformat the content into clear, well-structured markdown + """, + chunk_token_threshold=4096 +) +``` +> **Performance Tip**: Set a smaller `chunk_token_threshold` (e.g., 2048 or 4096) to enable parallel processing of content chunks. The default value is infinity, which processes the entire content as a single chunk. +## 6. Using Fit Markdown +When a content filter is active, the library produces two forms of markdown inside `result.markdown`: +1. **`raw_markdown`**: The full unfiltered markdown. +2. **`fit_markdown`**: A “fit” version where the filter has removed or trimmed noisy segments. +```python +import asyncio +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig +from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator +from crawl4ai.content_filter_strategy import PruningContentFilter + +async def main(): + config = CrawlerRunConfig( + markdown_generator=DefaultMarkdownGenerator( + content_filter=PruningContentFilter(threshold=0.6), + options={"ignore_links": True} + ) + ) + async with AsyncWebCrawler() as crawler: + result = await crawler.arun("https://news.example.com/tech", config=config) + if result.success: + print("Raw markdown:\n", result.markdown) + + # If a filter is used, we also have .fit_markdown: + md_object = result.markdown # or your equivalent + print("Filtered markdown:\n", md_object.fit_markdown) + else: + print("Crawl failed:", result.error_message) + +if __name__ == "__main__": + asyncio.run(main()) +``` +## 7. The `MarkdownGenerationResult` Object +If your library stores detailed markdown output in an object like `MarkdownGenerationResult`, you’ll see fields such as: +- **`raw_markdown`**: The direct HTML-to-markdown transformation (no filtering). +- **`markdown_with_citations`**: A version that moves links to reference-style footnotes. +- **`references_markdown`**: A separate string or section containing the gathered references. +- **`fit_markdown`**: The filtered markdown if you used a content filter. +- **`fit_html`**: The corresponding HTML snippet used to generate `fit_markdown` (helpful for debugging or advanced usage). +```python +md_obj = result.markdown # your library’s naming may vary +print("RAW:\n", md_obj.raw_markdown) +print("CITED:\n", md_obj.markdown_with_citations) +print("REFERENCES:\n", md_obj.references_markdown) +print("FIT:\n", md_obj.fit_markdown) +``` +- You can supply `raw_markdown` to an LLM if you want the entire text. +- Or feed `fit_markdown` into a vector database to reduce token usage. +- `references_markdown` can help you keep track of link provenance. +## 8. Combining Filters (BM25 + Pruning) in Two Passes +You might want to **prune out** noisy boilerplate first (with `PruningContentFilter`), and then **rank what’s left** against a user query (with `BM25ContentFilter`). You don’t have to crawl the page twice. Instead: +1. **First pass**: Apply `PruningContentFilter` directly to the raw HTML from `result.html` (the crawler’s downloaded HTML). +2. **Second pass**: Take the pruned HTML (or text) from step 1, and feed it into `BM25ContentFilter`, focusing on a user query. +### Two-Pass Example +```python +import asyncio +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig +from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter +from bs4 import BeautifulSoup + +async def main(): + # 1. Crawl with minimal or no markdown generator, just get raw HTML + config = CrawlerRunConfig( + # If you only want raw HTML, you can skip passing a markdown_generator + # or provide one but focus on .html in this example + ) + + async with AsyncWebCrawler() as crawler: + result = await crawler.arun("https://example.com/tech-article", config=config) + + if not result.success or not result.html: + print("Crawl failed or no HTML content.") + return + + raw_html = result.html + + # 2. First pass: PruningContentFilter on raw HTML + pruning_filter = PruningContentFilter(threshold=0.5, min_word_threshold=50) + + # filter_content returns a list of "text chunks" or cleaned HTML sections + pruned_chunks = pruning_filter.filter_content(raw_html) + # This list is basically pruned content blocks, presumably in HTML or text form + + # For demonstration, let's combine these chunks back into a single HTML-like string + # or you could do further processing. It's up to your pipeline design. + pruned_html = "\n".join(pruned_chunks) + + # 3. Second pass: BM25ContentFilter with a user query + bm25_filter = BM25ContentFilter( + user_query="machine learning", + bm25_threshold=1.2, + language="english" + ) + + # returns a list of text chunks + bm25_chunks = bm25_filter.filter_content(pruned_html) + + if not bm25_chunks: + print("Nothing matched the BM25 query after pruning.") + return + + # 4. Combine or display final results + final_text = "\n---\n".join(bm25_chunks) + + print("==== PRUNED OUTPUT (first pass) ====") + print(pruned_html[:500], "... (truncated)") # preview + + print("\n==== BM25 OUTPUT (second pass) ====") + print(final_text[:500], "... (truncated)") + +if __name__ == "__main__": + asyncio.run(main()) +``` +### What’s Happening? +1. **Raw HTML**: We crawl once and store the raw HTML in `result.html`. +4. **BM25ContentFilter**: We feed the pruned string into `BM25ContentFilter` with a user query. This second pass further narrows the content to chunks relevant to “machine learning.” +**No Re-Crawling**: We used `raw_html` from the first pass, so there’s no need to run `arun()` again—**no second network request**. +### Tips & Variations +- **Plain Text vs. HTML**: If your pruned output is mostly text, BM25 can still handle it; just keep in mind it expects a valid string input. If you supply partial HTML (like `"

some text

"`), it will parse it as HTML. +- **Adjust Thresholds**: If you see too much or too little text in step one, tweak `threshold=0.5` or `min_word_threshold=50`. Similarly, `bm25_threshold=1.2` can be raised/lowered for more or fewer chunks in step two. +### One-Pass Combination? +## 9. Common Pitfalls & Tips +1. **No Markdown Output?** +2. **Performance Considerations** + - Very large pages with multiple filters can be slower. Consider `cache_mode` to avoid re-downloading. +3. **Take Advantage of `fit_markdown`** +4. **Adjusting `html2text` Options** + - If you see lots of raw HTML slipping into the text, turn on `escape_html`. + - If code blocks look messy, experiment with `mark_code` or `handle_code_in_pre`. +## 10. Summary & Next Steps +- Configure the **DefaultMarkdownGenerator** with HTML-to-text options. +- Select different HTML sources using the `content_source` parameter. +- Distinguish between raw and filtered markdown (`fit_markdown`). +- Leverage the `MarkdownGenerationResult` object to handle different forms of output (citations, references, etc.). + + +# Fit Markdown with Pruning & BM25 +## 1. How “Fit Markdown” Works +### 1.1 The `content_filter` +In **`CrawlerRunConfig`**, you can specify a **`content_filter`** to shape how content is pruned or ranked before final markdown generation. A filter’s logic is applied **before** or **during** the HTML→Markdown process, producing: +- **`result.markdown.raw_markdown`** (unfiltered) +- **`result.markdown.fit_markdown`** (filtered or “fit” version) +- **`result.markdown.fit_html`** (the corresponding HTML snippet that produced `fit_markdown`) +### 1.2 Common Filters +## 2. PruningContentFilter +### 2.1 Usage Example +```python +import asyncio +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig +from crawl4ai.content_filter_strategy import PruningContentFilter +from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator + +async def main(): + # Step 1: Create a pruning filter + prune_filter = PruningContentFilter( + # Lower → more content retained, higher → more content pruned + threshold=0.45, + # "fixed" or "dynamic" + threshold_type="dynamic", + # Ignore nodes with <5 words + min_word_threshold=5 + ) + + # Step 2: Insert it into a Markdown Generator + md_generator = DefaultMarkdownGenerator(content_filter=prune_filter) + + # Step 3: Pass it to CrawlerRunConfig + config = CrawlerRunConfig( + markdown_generator=md_generator + ) + + async with AsyncWebCrawler() as crawler: + result = await crawler.arun( + url="https://news.ycombinator.com", + config=config + ) + + if result.success: + # 'fit_markdown' is your pruned content, focusing on "denser" text + print("Raw Markdown length:", len(result.markdown.raw_markdown)) + print("Fit Markdown length:", len(result.markdown.fit_markdown)) + else: + print("Error:", result.error_message) + +if __name__ == "__main__": + asyncio.run(main()) +``` +### 2.2 Key Parameters +- **`min_word_threshold`** (int): If a block has fewer words than this, it’s pruned. +- **`threshold_type`** (str): + - `"fixed"` → each node must exceed `threshold` (0–1). + - `"dynamic"` → node scoring adjusts according to tag type, text/link density, etc. +- **`threshold`** (float, default ~0.48): The base or “anchor” cutoff. +- **Link density** – Penalizes sections that are mostly links. +- **Tag importance** – e.g., an `
` or `

` might be more important than a `

`. +## 3. BM25ContentFilter +### 3.1 Usage Example +```python +import asyncio +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig +from crawl4ai.content_filter_strategy import BM25ContentFilter +from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator + +async def main(): + # 1) A BM25 filter with a user query + bm25_filter = BM25ContentFilter( + user_query="startup fundraising tips", + # Adjust for stricter or looser results + bm25_threshold=1.2 + ) + + # 2) Insert into a Markdown Generator + md_generator = DefaultMarkdownGenerator(content_filter=bm25_filter) + + # 3) Pass to crawler config + config = CrawlerRunConfig( + markdown_generator=md_generator + ) + + async with AsyncWebCrawler() as crawler: + result = await crawler.arun( + url="https://news.ycombinator.com", + config=config + ) + if result.success: + print("Fit Markdown (BM25 query-based):") + print(result.markdown.fit_markdown) + else: + print("Error:", result.error_message) + +if __name__ == "__main__": + asyncio.run(main()) +``` +### 3.2 Parameters +- **`user_query`** (str, optional): E.g. `"machine learning"`. If blank, the filter tries to glean a query from page metadata. +- **`bm25_threshold`** (float, default 1.0): + - Higher → fewer chunks but more relevant. + - Lower → more inclusive. +> In more advanced scenarios, you might see parameters like `language`, `case_sensitive`, or `priority_tags` to refine how text is tokenized or weighted. +## 4. Accessing the “Fit” Output +After the crawl, your “fit” content is found in **`result.markdown.fit_markdown`**. +```python +fit_md = result.markdown.fit_markdown +fit_html = result.markdown.fit_html +``` +If the content filter is **BM25**, you might see additional logic or references in `fit_markdown` that highlight relevant segments. If it’s **Pruning**, the text is typically well-cleaned but not necessarily matched to a query. +## 5. Code Patterns Recap +### 5.1 Pruning +```python +prune_filter = PruningContentFilter( + threshold=0.5, + threshold_type="fixed", + min_word_threshold=10 +) +md_generator = DefaultMarkdownGenerator(content_filter=prune_filter) +config = CrawlerRunConfig(markdown_generator=md_generator) +``` +### 5.2 BM25 +```python +bm25_filter = BM25ContentFilter( + user_query="health benefits fruit", + bm25_threshold=1.2 +) +md_generator = DefaultMarkdownGenerator(content_filter=bm25_filter) +config = CrawlerRunConfig(markdown_generator=md_generator) +``` +## 6. Combining with “word_count_threshold” & Exclusions +```python +config = CrawlerRunConfig( + word_count_threshold=10, + excluded_tags=["nav", "footer", "header"], + exclude_external_links=True, + markdown_generator=DefaultMarkdownGenerator( + content_filter=PruningContentFilter(threshold=0.5) + ) +) +``` +1. The crawler’s `excluded_tags` are removed from the HTML first. +3. The final “fit” content is generated in `result.markdown.fit_markdown`. +## 7. Custom Filters +If you need a different approach (like a specialized ML model or site-specific heuristics), you can create a new class inheriting from `RelevantContentFilter` and implement `filter_content(html)`. Then inject it into your **markdown generator**: +```python +from crawl4ai.content_filter_strategy import RelevantContentFilter + +class MyCustomFilter(RelevantContentFilter): + def filter_content(self, html, min_word_threshold=None): + # parse HTML, implement custom logic + return [block for block in ... if ... some condition...] + +``` +1. Subclass `RelevantContentFilter`. +2. Implement `filter_content(...)`. +3. Use it in your `DefaultMarkdownGenerator(content_filter=MyCustomFilter(...))`. +## 8. Final Thoughts +- **Summaries**: Quickly get the important text from a cluttered page. +- **Search**: Combine with **BM25** to produce content relevant to a query. +- **BM25ContentFilter**: Perfect for query-based extraction or searching. +- Combine with **`excluded_tags`, `exclude_external_links`, `word_count_threshold`** to refine your final “fit” text. +- Fit markdown ends up in **`result.markdown.fit_markdown`**; eventually **`result.markdown.fit_markdown`** in future versions. +- Last Updated: 2025-01-01 + + +# Content Selection +Crawl4AI provides multiple ways to **select**, **filter**, and **refine** the content from your crawls. Whether you need to target a specific CSS region, exclude entire tags, filter out external links, or remove certain domains and images, **`CrawlerRunConfig`** offers a wide range of parameters. +## 1. CSS-Based Selection +There are two ways to select content from a page: using `css_selector` or the more flexible `target_elements`. +### 1.1 Using `css_selector` +A straightforward way to **limit** your crawl results to a certain region of the page is **`css_selector`** in **`CrawlerRunConfig`**: +```python +import asyncio +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig + +async def main(): + config = CrawlerRunConfig( + # e.g., first 30 items from Hacker News + css_selector=".athing:nth-child(-n+30)" + ) + async with AsyncWebCrawler() as crawler: + result = await crawler.arun( + url="https://news.ycombinator.com/newest", + config=config + ) + print("Partial HTML length:", len(result.cleaned_html)) + +if __name__ == "__main__": + asyncio.run(main()) +``` +**Result**: Only elements matching that selector remain in `result.cleaned_html`. +### 1.2 Using `target_elements` +The `target_elements` parameter provides more flexibility by allowing you to target **multiple elements** for content extraction while preserving the entire page context for other features: +```python +import asyncio +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig + +async def main(): + config = CrawlerRunConfig( + # Target article body and sidebar, but not other content + target_elements=["article.main-content", "aside.sidebar"] + ) + async with AsyncWebCrawler() as crawler: + result = await crawler.arun( + url="https://example.com/blog-post", + config=config + ) + print("Markdown focused on target elements") + print("Links from entire page still available:", len(result.links.get("internal", []))) + +if __name__ == "__main__": + asyncio.run(main()) +``` +**Key difference**: With `target_elements`, the markdown generation and structural data extraction focus on those elements, but other page elements (like links, images, and tables) are still extracted from the entire page. This gives you fine-grained control over what appears in your markdown content while preserving full page context for link analysis and media collection. +## 2. Content Filtering & Exclusions +### 2.1 Basic Overview +```python +config = CrawlerRunConfig( + # Content thresholds + word_count_threshold=10, # Minimum words per block + + # Tag exclusions + excluded_tags=['form', 'header', 'footer', 'nav'], + + # Link filtering + exclude_external_links=True, + exclude_social_media_links=True, + # Block entire domains + exclude_domains=["adtrackers.com", "spammynews.org"], + exclude_social_media_domains=["facebook.com", "twitter.com"], + + # Media filtering + exclude_external_images=True +) +``` +- **`word_count_threshold`**: Ignores text blocks under X words. Helps skip trivial blocks like short nav or disclaimers. +- **`excluded_tags`**: Removes entire tags (``, `
`, `