ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: Add support for GPU, MPS, and CPU

Browse Source
This commit is contained in:
unclecode
2024-05-17 21:56:13 +08:00
parent 0a902f562f
commit b6319c6f6e
5 changed files with 20 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
docs/examples/quickstart.py
View File

@@ -12,7 +12,7 @@ console = Console()
@lru_cache()
def create_crawler():
crawler = WebCrawler()
crawler = WebCrawler(verbose=True)
crawler.warmup()
return crawler
@@ -86,7 +86,7 @@ def add_extraction_strategy(crawler):
cprint("CosineStrategy uses cosine similarity to extract semantically similar blocks of text. Let's see it in action!")
result = crawler.run(
url="https://www.nbcnews.com/business",
extraction_strategy=CosineStrategy(word_count_threshold=10, max_dist=0.2, linkage_method="ward", top_k=3)
extraction_strategy=CosineStrategy(word_count_threshold=10, max_dist=0.2, linkage_method="ward", top_k=3, verbose=True)
)
cprint("[LOG] 📦 [bold yellow]CosineStrategy result:[/bold yellow]")
print_result(result)
@@ -171,7 +171,7 @@ def main():
crawler = create_crawler()
basic_usage(crawler)
# basic_usage(crawler)
understanding_parameters(crawler)
crawler.always_by_pass_cache = True