3d00fee6c2
- Another thing this commit introduces is the concept of the Relevance Content Filter. This is an improvement over Fit Markdown. This class of strategies aims to extract the main content from a given page - the part that really matters and is useful to be processed. One strategy has been created using the BM25 algorithm, which finds chunks of text from the web page relevant to its title, descriptions, and keywords, or supports a given user query and matches them. The result is then returned to the main engine to be converted to Markdown. Plans include adding approaches using language models as well. - The cache database was updated to hold information about response headers and downloaded files.
39 lines
1.1 KiB
Python
39 lines
1.1 KiB
Python
from pydantic import BaseModel, HttpUrl
|
|
from typing import List, Dict, Optional, Callable, Awaitable
|
|
|
|
|
|
|
|
class UrlModel(BaseModel):
|
|
url: HttpUrl
|
|
forced: bool = False
|
|
|
|
class CrawlResult(BaseModel):
|
|
url: str
|
|
html: str
|
|
success: bool
|
|
cleaned_html: Optional[str] = None
|
|
media: Dict[str, List[Dict]] = {}
|
|
links: Dict[str, List[Dict]] = {}
|
|
downloaded_files: Optional[List[str]] = None
|
|
screenshot: Optional[str] = None
|
|
markdown: Optional[str] = None
|
|
fit_markdown: Optional[str] = None
|
|
fit_html: Optional[str] = None
|
|
extracted_content: Optional[str] = None
|
|
metadata: Optional[dict] = None
|
|
error_message: Optional[str] = None
|
|
session_id: Optional[str] = None
|
|
response_headers: Optional[dict] = None
|
|
status_code: Optional[int] = None
|
|
|
|
class AsyncCrawlResponse(BaseModel):
|
|
html: str
|
|
response_headers: Dict[str, str]
|
|
status_code: int
|
|
screenshot: Optional[str] = None
|
|
get_delayed_content: Optional[Callable[[Optional[float]], Awaitable[str]]] = None
|
|
downloaded_files: Optional[List[str]] = None
|
|
|
|
class Config:
|
|
arbitrary_types_allowed = True
|