Enhance AsyncWebCrawler and related configurations

- Introduced new configuration classes: BrowserConfig and CrawlerRunConfig.
  - Refactored AsyncWebCrawler to leverage the new configuration system for cleaner parameter management.
  - Updated AsyncPlaywrightCrawlerStrategy for better flexibility and reduced legacy parameters.
  - Improved error handling with detailed context extraction during exceptions.
  - Enhanced overall maintainability and usability of the web crawler.

crawl4ai/async_database.py

@@ -1,4 +1,4 @@
-import os
+import os, sys
 from pathlib import Path
 import aiosqlite
 import asyncio
@@ -13,6 +13,7 @@ import aiofiles
 from .config import NEED_MIGRATION
 from .version_manager import VersionManager
 from .async_logger import AsyncLogger
+from .utils import get_error_context, create_box_message
 # Set up logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -97,35 +98,84 @@ class AsyncDatabaseManager:
 
     @asynccontextmanager
     async def get_connection(self):
-        """Connection pool manager"""
+        """Connection pool manager with enhanced error handling"""
         if not self._initialized:
-            # Use an asyncio.Lock to ensure only one initialization occurs
             async with self.init_lock:
                 if not self._initialized:
-                    await self.initialize()
-                    self._initialized = True
+                    try:
+                        await self.initialize()
+                        self._initialized = True
+                    except Exception as e:
+                        import sys
+                        error_context = get_error_context(sys.exc_info())
+                        self.logger.error(
+                            message="Database initialization failed:\n{error}\n\nContext:\n{context}\n\nTraceback:\n{traceback}",
+                            tag="ERROR",
+                            force_verbose=True,
+                            params={
+                                "error": str(e),
+                                "context": error_context["code_context"],
+                                "traceback": error_context["full_traceback"]
+                            }
+                        )
+                        raise
 
         await self.connection_semaphore.acquire()
         task_id = id(asyncio.current_task())
+        
         try:
             async with self.pool_lock:
                 if task_id not in self.connection_pool:
-                    conn = await aiosqlite.connect(
-                        self.db_path,
-                        timeout=30.0
-                    )
-                    await conn.execute('PRAGMA journal_mode = WAL')
-                    await conn.execute('PRAGMA busy_timeout = 5000')
-                    self.connection_pool[task_id] = conn
+                    try:
+                        conn = await aiosqlite.connect(
+                            self.db_path,
+                            timeout=30.0
+                        )
+                        await conn.execute('PRAGMA journal_mode = WAL')
+                        await conn.execute('PRAGMA busy_timeout = 5000')
+                        
+                        # Verify database structure
+                        async with conn.execute("PRAGMA table_info(crawled_data)") as cursor:
+                            columns = await cursor.fetchall()
+                            column_names = [col[1] for col in columns]
+                            expected_columns = {
+                                'url', 'html', 'cleaned_html', 'markdown', 'extracted_content',
+                                'success', 'media', 'links', 'metadata', 'screenshot',
+                                'response_headers', 'downloaded_files'
+                            }
+                            missing_columns = expected_columns - set(column_names)
+                            if missing_columns:
+                                raise ValueError(f"Database missing columns: {missing_columns}")
+                        
+                        self.connection_pool[task_id] = conn
+                    except Exception as e:
+                        import sys
+                        error_context = get_error_context(sys.exc_info())
+                        error_message = (
+                            f"Unexpected error in db get_connection at line {error_context['line_no']} "
+                            f"in {error_context['function']} ({error_context['filename']}):\n"
+                            f"Error: {str(e)}\n\n"
+                            f"Code context:\n{error_context['code_context']}"
+                        )
+                        self.logger.error(
+                            message=create_box_message(error_message, type= "error"),
+                        )
+
+                        raise
 
             yield self.connection_pool[task_id]
 
         except Exception as e:
+            import sys
+            error_context = get_error_context(sys.exc_info())
+            error_message = (
+                f"Unexpected error in db get_connection at line {error_context['line_no']} "
+                f"in {error_context['function']} ({error_context['filename']}):\n"
+                f"Error: {str(e)}\n\n"
+                f"Code context:\n{error_context['code_context']}"
+            )
             self.logger.error(
-                message="Connection error: {error}",
-                tag="ERROR",
-                force_verbose=True,
-                params={"error": str(e)}
+                message=create_box_message(error_message, type= "error"),
             )
             raise
         finally:
@@ -230,7 +280,8 @@ class AsyncDatabaseManager:
                     'cleaned_html': row_dict['cleaned_html'],
                     'markdown': row_dict['markdown'],
                     'extracted_content': row_dict['extracted_content'],
-                    'screenshot': row_dict['screenshot']
+                    'screenshot': row_dict['screenshot'],
+                    'screenshots': row_dict['screenshot'],
                 }
                 
                 for field, hash_value in content_fields.items():