ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update Readme: Showcase the speed

Browse Source
This commit is contained in:
unclecode
2024-06-24 23:02:08 +08:00
parent f51b078042
commit 1fffeeedd2
2 changed files with 31 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
crawl4ai/web_crawler.py
View File

@@ -189,7 +189,8 @@ class WebCrawler:
# print(f"[LOG] 🚀 Crawling done for {url}, success: True, time taken: {time.time() - t1} seconds")
t1 = time.time()
result = get_content_of_website_optimized(url, html, word_count_threshold, css_selector=css_selector, only_text=kwargs.get("only_text", False))
print(f"[LOG] 🚀 Crawling done for {url}, success: True, time taken: {time.time() - t1} seconds")
if verbose:
print(f"[LOG] 🚀 Crawling done for {url}, success: True, time taken: {time.time() - t1} seconds")
if result is None:
raise ValueError(f"Failed to extract content from the website: {url}")
@@ -201,9 +202,6 @@ class WebCrawler:
media = result.get("media", [])
links = result.get("links", [])
metadata = result.get("metadata", {})
if verbose:
print(f"[LOG] 🚀 Crawling done for {url}, success: True, time taken: {time.time() - t} seconds")
if extracted_content is None:
if verbose: