ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: remove trailing slash when the path is empty. This is causing dupicate crawls

Browse Source
This commit is contained in:
Aravind Karnam
2025-03-21 13:41:31 +05:30
parent 8b761f232b
commit 6740e87b4d
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crawl4ai/utils.py
View File

@@ -2002,7 +2002,7 @@ def normalize_url_for_deep_crawl(href, base_url):
normalized = urlunparse(( normalized = urlunparse((
parsed.scheme, parsed.scheme,
netloc, netloc,
parsed.path.rstrip('/') or '/', # Normalize trailing slash parsed.path.rstrip('/'), # Normalize trailing slash
parsed.params, parsed.params,
query, query,
fragment fragment
@@ -2030,7 +2030,7 @@ def efficient_normalize_url_for_deep_crawl(href, base_url):
normalized = urlunparse(( normalized = urlunparse((
parsed.scheme, parsed.scheme,
parsed.netloc.lower(), parsed.netloc.lower(),
parsed.path, parsed.path.rstrip('/'),
parsed.params, parsed.params,
parsed.query, parsed.query,
'' # Remove fragment '' # Remove fragment