ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix merge_chunks splitter usage and remove incorrect return

Browse Source
This commit is contained in:
UncleCode
2025-05-17 10:31:19 +08:00
parent 897e017361
commit 45f1652d98
1 changed files with 1 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crawl4ai/utils.py
View File

@@ -150,7 +150,7 @@ def merge_chunks(
total_tokens = 0 total_tokens = 0
for doc in docs: for doc in docs:
tokens = doc.split() tokens = splitter(doc)
count = int(len(tokens) * word_token_ratio) count = int(len(tokens) * word_token_ratio)
if count: # Skip empty docs if count: # Skip empty docs
token_counts.append(count) token_counts.append(count)
@@ -1196,8 +1196,6 @@ def get_content_of_website_optimized(
return None return None
except InvalidSchema: except InvalidSchema:
return None return None
finally:
return
image_height = img.get("height") image_height = img.get("height")
height_value, height_unit = parse_dimension(image_height) height_value, height_unit = parse_dimension(image_height)