chore(cleanup): remove unused files and improve type hints

- Remove .pre-commit-config.yaml and duplicate mkdocs configuration files
- Add Optional type hint for proxy parameter in BrowserConfig
- Fix type annotation for results list in AsyncWebCrawler
- Move calculate_batch_size function import to model_loader
- Update prompt imports in extraction_strategy.py

No breaking changes.
This commit is contained in:
UncleCode
2025-01-14 13:07:18 +08:00
parent 8ec12d7d68
commit 20c027b79c
6 changed files with 8 additions and 208 deletions

View File

@@ -5,20 +5,17 @@ import json
import time
import os
from .prompts import PROMPT_EXTRACT_BLOCKS
from .prompts import PROMPT_EXTRACT_BLOCKS, PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION, PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION
from .config import (
DEFAULT_PROVIDER, PROVIDER_MODELS,
CHUNK_TOKEN_THRESHOLD,
OVERLAP_RATE,
WORD_TOKEN_RATE,
PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION,
PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION
)
from .utils import * # noqa: F403
from .utils import (
sanitize_html,
calculate_batch_size,
escape_json_string,
perform_completion_with_backoff,
extract_xml_data,
@@ -34,6 +31,7 @@ from .model_loader import (
get_device,
load_HF_embedding_model,
load_text_multilabel_classifier,
calculate_batch_size
)
from functools import partial