Commit Message:

Enhance crawler capabilities and documentation

  - Added SSL certificate extraction in AsyncWebCrawler.
  - Introduced new content filters and chunking strategies for more robust data extraction.
  - Updated documentation management to streamline user experience.

crawl4ai/models.py

@@ -1,6 +1,8 @@
 from pydantic import BaseModel, HttpUrl
 from typing import List, Dict, Optional, Callable, Awaitable, Union, Any
 from dataclasses import dataclass
+from .ssl_certificate import SSLCertificate
+
 @dataclass
 class TokenUsage:
     completion_tokens: int = 0
@@ -41,7 +43,9 @@ class CrawlResult(BaseModel):
     session_id: Optional[str] = None
     response_headers: Optional[dict] = None
     status_code: Optional[int] = None
-    ssl_certificate: Optional[Dict[str, Any]] = None
+    ssl_certificate: Optional[SSLCertificate] = None
+    class Config:
+        arbitrary_types_allowed = True
 
 class AsyncCrawlResponse(BaseModel):
     html: str
@@ -51,7 +55,7 @@ class AsyncCrawlResponse(BaseModel):
     pdf_data: Optional[bytes] = None
     get_delayed_content: Optional[Callable[[Optional[float]], Awaitable[str]]] = None
     downloaded_files: Optional[List[str]] = None
-    ssl_certificate: Optional[Dict[str, Any]] = None
+    ssl_certificate: Optional[SSLCertificate] = None
 
     class Config:
         arbitrary_types_allowed = True