ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: url normalisation ref: https://github.com/unclecode/crawl4ai/issues/841

Browse Source
This commit is contained in:
Aravind Karnam
2025-03-21 16:48:07 +05:30
parent f89113377a
commit 471d110c5e
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crawl4ai/utils.py
View File

@@ -1958,6 +1958,10 @@ def normalize_url(href, base_url):
if not parsed_base.scheme or not parsed_base.netloc: if not parsed_base.scheme or not parsed_base.netloc:
raise ValueError(f"Invalid base URL format: {base_url}") raise ValueError(f"Invalid base URL format: {base_url}")
# Ensure base_url ends with a trailing slash if it's a directory path
if not base_url.endswith('/'):
base_url = base_url + '/'
# Use urljoin to handle all cases # Use urljoin to handle all cases
normalized = urljoin(base_url, href.strip()) normalized = urljoin(base_url, href.strip())
return normalized return normalized