chore(cleanup): remove unused files and improve type hints

- Remove .pre-commit-config.yaml and duplicate mkdocs configuration files
- Add Optional type hint for proxy parameter in BrowserConfig
- Fix type annotation for results list in AsyncWebCrawler
- Move calculate_batch_size function import to model_loader
- Update prompt imports in extraction_strategy.py

No breaking changes.
This commit is contained in:
UncleCode
2025-01-14 13:07:18 +08:00
parent 8ec12d7d68
commit 20c027b79c
6 changed files with 8 additions and 208 deletions

View File

@@ -10,7 +10,7 @@ import asyncio
# from contextlib import nullcontext, asynccontextmanager
from contextlib import asynccontextmanager
from .models import CrawlResult, MarkdownGenerationResult, CrawlerTaskResult, DispatchResult, RateLimiter
from .models import CrawlResult, MarkdownGenerationResult, CrawlerTaskResult, DispatchResult
from .async_database import async_db_manager
from .chunking_strategy import * # noqa: F403
from .chunking_strategy import RegexChunking, ChunkingStrategy, IdentityChunking
@@ -31,7 +31,7 @@ from .markdown_generation_strategy import (
from .async_logger import AsyncLogger
from .async_configs import BrowserConfig, CrawlerRunConfig
from .async_dispatcher import * # noqa: F403
from .async_dispatcher import BaseDispatcher, MemoryAdaptiveDispatcher
from .async_dispatcher import BaseDispatcher, MemoryAdaptiveDispatcher, RateLimiter
from .config import MIN_WORD_THRESHOLD
from .utils import (
@@ -783,7 +783,7 @@ class AsyncWebCrawler:
crawler=self, urls=urls, config=config
)
results: CrawlResult = []
results: List[CrawlResult] = []
for res in _results:
_res: CrawlResult = res.result
dispatch_result: DispatchResult = DispatchResult(