6ec4cb33ca
- Integrate customized html2text library for flexible Markdown output - Add options to exclude external links and images - Improve content scraping efficiency and error handling - Update AsyncPlaywrightCrawlerStrategy for faster closing - Enhance CosineStrategy with generic embedding model loading
19 lines
423 B
Python
19 lines
423 B
Python
from typing import Dict, Optional
|
|
|
|
|
|
class AnchorElement:
|
|
__slots__ = ["attrs", "count", "outcount"]
|
|
|
|
def __init__(self, attrs: Dict[str, Optional[str]], count: int, outcount: int):
|
|
self.attrs = attrs
|
|
self.count = count
|
|
self.outcount = outcount
|
|
|
|
|
|
class ListElement:
|
|
__slots__ = ["name", "num"]
|
|
|
|
def __init__(self, name: str, num: int):
|
|
self.name = name
|
|
self.num = num
|