fix(auth): make JWT authentication optional with fallback

Modify authentication system to gracefully handle cases where JWT is not enabled or token is missing. This includes:
- Making HTTPBearer auto_error=False to prevent automatic 403 errors
- Updating token dependency to return None when JWT is disabled
- Fixing model deserialization in CrawlResult
- Updating documentation links
- Cleaning up imports

BREAKING CHANGE: Authentication behavior changed to be more permissive when JWT is disabled

deploy/docker/README.md

@@ -595,8 +595,8 @@ curl http://localhost:8000/health
 ## Complete Examples
 
 Check out the `examples` folder in our repository for full working examples! Here are two to get you started:
-[Using Client SDK](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_python_sdk_example.py)
-[Using REST API](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_python_rest_api_example.py)
+[Using Client SDK](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_python_sdk.py)
+[Using REST API](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_python_rest_api.py)
 
 ## Server Configuration
 

deploy/docker/auth.py

@@ -10,7 +10,7 @@ from pydantic.main import BaseModel
 import base64
 
 instance = JWT()
-security = HTTPBearer()
+security = HTTPBearer(auto_error=False)
 SECRET_KEY = os.environ.get("SECRET_KEY", "mysecret")
 ACCESS_TOKEN_EXPIRE_MINUTES = 60
 
@@ -30,6 +30,9 @@ def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) -
 
 def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)) -> Dict:
     """Verify the JWT token from the Authorization header."""
+
+    if credentials is None:
+        return None
     token = credentials.credentials
     verifying_key = get_jwk_from_secret(SECRET_KEY)
     try:
@@ -38,9 +41,15 @@ def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security))
     except Exception:
         raise HTTPException(status_code=401, detail="Invalid or expired token")
 
+
 def get_token_dependency(config: Dict):
-    """Return the token dependency if JWT is enabled, else None."""
-    return verify_token if config.get("security", {}).get("jwt_enabled", False) else None
+    """Return the token dependency if JWT is enabled, else a function that returns None."""
+
+    if config.get("security", {}).get("jwt_enabled", False):
+        return verify_token
+    else:
+        return lambda: None
+
 
 class TokenRequest(BaseModel):
     email: EmailStr

deploy/docker/server.py

@@ -92,7 +92,7 @@ async def get_markdown(
     f: FilterType = FilterType.FIT,
     q: Optional[str] = None,
     c: Optional[str] = "0",
-    # token_data: Optional[Dict] = Depends(token_dependency)
+    token_data: Optional[Dict] = Depends(token_dependency)
 ):
     result = await handle_markdown_request(url, f, q, c, config)
     return PlainTextResponse(result)