fix(auth): make JWT authentication optional with fallback

Modify authentication system to gracefully handle cases where JWT is not enabled or token is missing. This includes:
- Making HTTPBearer auto_error=False to prevent automatic 403 errors
- Updating token dependency to return None when JWT is disabled
- Fixing model deserialization in CrawlResult
- Updating documentation links
- Cleaning up imports

BREAKING CHANGE: Authentication behavior changed to be more permissive when JWT is disabled

crawl4ai/async_database.py

@@ -4,10 +4,10 @@ import aiosqlite
 import asyncio
 from typing import Optional, Dict
 from contextlib import asynccontextmanager
-import logging
 import json  # Added for serialization/deserialization
 from .utils import ensure_content_dirs, generate_content_hash
 from .models import CrawlResult, MarkdownGenerationResult, StringCompatibleMarkdown
+# , StringCompatibleMarkdown
 import aiofiles
 from .utils import VersionManager
 from .async_logger import AsyncLogger

crawl4ai/markdown_generation_strategy.py

@@ -2,8 +2,8 @@ from abc import ABC, abstractmethod
 from typing import Optional, Dict, Any, Tuple
 from .models import MarkdownGenerationResult
 from .html2text import CustomHTML2Text
-from .types import RelevantContentFilter
-# from .content_filter_strategy import RelevantContentFilter
+# from .types import RelevantContentFilter
+from .content_filter_strategy import RelevantContentFilter
 import re
 from urllib.parse import urljoin
 

crawl4ai/models.py

@@ -149,7 +149,11 @@ class CrawlResult(BaseModel):
         markdown_result = data.pop('markdown', None)
         super().__init__(**data)
         if markdown_result is not None:
-            self._markdown = markdown_result
+            self._markdown = (
+                MarkdownGenerationResult(**markdown_result)
+                if isinstance(markdown_result, dict)
+                else markdown_result
+            )
     
     @property
     def markdown(self):