4750810a67
- Implement smart_wait function in AsyncPlaywrightCrawlerStrategy - Add screenshot support to AsyncCrawlResponse and AsyncWebCrawler - Improve error handling and timeout management in crawling process - Fix typo in CrawlResult model (responser_headers -> response_headers) - Update .gitignore to exclude additional files - Adjust import path in test_basic_crawling.py
22 lines
651 B
Python
22 lines
651 B
Python
from pydantic import BaseModel, HttpUrl
|
|
from typing import List, Dict, Optional
|
|
|
|
class UrlModel(BaseModel):
|
|
url: HttpUrl
|
|
forced: bool = False
|
|
|
|
class CrawlResult(BaseModel):
|
|
url: str
|
|
html: str
|
|
success: bool
|
|
cleaned_html: Optional[str] = None
|
|
media: Dict[str, List[Dict]] = {}
|
|
links: Dict[str, List[Dict]] = {}
|
|
screenshot: Optional[str] = None
|
|
markdown: Optional[str] = None
|
|
extracted_content: Optional[str] = None
|
|
metadata: Optional[dict] = None
|
|
error_message: Optional[str] = None
|
|
session_id: Optional[str] = None
|
|
response_headers: Optional[dict] = None
|
|
status_code: Optional[int] = None |