ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update web_crawler.py

Browse Source 
Set CosineExtraction as defaul strategy
This commit is contained in:
UncleCode
2024-05-16 22:28:24 +08:00
committed by GitHub
parent a856319499
commit 5b4a586b2d
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
crawl4ai/web_crawler.py
View File

@@ -63,7 +63,7 @@ class WebCrawler:
extract_blocks_flag: bool = True, extract_blocks_flag: bool = True,
word_count_threshold=MIN_WORD_THRESHOLD, word_count_threshold=MIN_WORD_THRESHOLD,
use_cached_html: bool = False, use_cached_html: bool = False,
extraction_strategy: ExtractionStrategy = NoExtractionStrategy(), extraction_strategy: ExtractionStrategy = CosineStrategy(),
chunking_strategy: ChunkingStrategy = RegexChunking(), chunking_strategy: ChunkingStrategy = RegexChunking(),
**kwargs, **kwargs,
) -> CrawlResult: ) -> CrawlResult:
@@ -82,7 +82,7 @@ class WebCrawler:
self, self,
url: str, url: str,
word_count_threshold=MIN_WORD_THRESHOLD, word_count_threshold=MIN_WORD_THRESHOLD,
extraction_strategy: ExtractionStrategy = NoExtractionStrategy(), extraction_strategy: ExtractionStrategy = CosineStrategy(),
chunking_strategy: ChunkingStrategy = RegexChunking(), chunking_strategy: ChunkingStrategy = RegexChunking(),
bypass_cache: bool = False, bypass_cache: bool = False,
css_selector: str = None, css_selector: str = None,
@@ -184,7 +184,7 @@ class WebCrawler:
extract_blocks_flag: bool = True, extract_blocks_flag: bool = True,
word_count_threshold=MIN_WORD_THRESHOLD, word_count_threshold=MIN_WORD_THRESHOLD,
use_cached_html: bool = False, use_cached_html: bool = False,
extraction_strategy: ExtractionStrategy = NoExtractionStrategy(), extraction_strategy: ExtractionStrategy = CosineStrategy(),
chunking_strategy: ChunkingStrategy = RegexChunking(), chunking_strategy: ChunkingStrategy = RegexChunking(),
**kwargs, **kwargs,
) -> List[CrawlResult]: ) -> List[CrawlResult]: