ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge PR #1717: Allow local embeddings by removing OpenAI fallback

Browse Source
This commit is contained in:
unclecode
2026-02-01 05:02:18 +00:00
parent 9172581416 2a04fc319a
commit 5cd0648d71
1 changed files with 5 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
crawl4ai/adaptive_crawler.py
View File

@@ -630,18 +630,15 @@ class EmbeddingStrategy(CrawlStrategy):
self._validation_embeddings_cache = None # Cache validation query embeddings self._validation_embeddings_cache = None # Cache validation query embeddings
self._kb_similarity_threshold = 0.95 # Threshold for deduplication self._kb_similarity_threshold = 0.95 # Threshold for deduplication
def _get_embedding_llm_config_dict(self) -> Dict: def _get_embedding_llm_config_dict(self) -> Optional[Dict]:
"""Get embedding LLM config as dict with fallback to default.""" """Get embedding LLM config as dict, or None for local embeddings."""
if hasattr(self, 'config') and self.config: if hasattr(self, 'config') and self.config:
config_dict = self.config._embedding_llm_config_dict config_dict = self.config._embedding_llm_config_dict
if config_dict: if config_dict:
return config_dict return config_dict
# Fallback to default if no config provided # Return None to use local sentence-transformers embeddings
return { return None
'provider': 'openai/text-embedding-3-small',
'api_token': os.getenv('OPENAI_API_KEY')
}
async def _get_embeddings(self, texts: List[str]) -> Any: async def _get_embeddings(self, texts: List[str]) -> Any:
"""Get embeddings using configured method""" """Get embeddings using configured method"""