chore: Refactor get_content_of_website_optimized function in utils.py

2024-06-26 14:43:09 +08:00
parent 96d1eb0d0d
commit 7ba2142363
2 changed files with 28 additions and 21 deletions
--- a/crawl4ai/web_crawler.py
+++ b/crawl4ai/web_crawler.py
@@ -136,8 +136,10 @@ class WebCrawler:
            if not isinstance(chunking_strategy, ChunkingStrategy):
                raise ValueError("Unsupported chunking strategy")
            
-            if word_count_threshold < MIN_WORD_THRESHOLD:
-                word_count_threshold = MIN_WORD_THRESHOLD
+            # if word_count_threshold < MIN_WORD_THRESHOLD:
+            #     word_count_threshold = MIN_WORD_THRESHOLD
+                
+            word_count_threshold = max(word_count_threshold, 0)

            # Check cache first
            cached = None