ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(utils): Improve URL normalization by avoiding quote/unquote to preserve '+' signs. ref #1332

Browse Source
This commit is contained in:
ntohidi
2025-08-22 12:05:21 +08:00
parent 90af453506
commit 40ab287c90
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
crawl4ai/utils.py
View File

@@ -2184,8 +2184,10 @@ def normalize_url(
netloc = parsed.netloc.lower() netloc = parsed.netloc.lower()
# ── path ── # ── path ──
# Strip duplicate slashes and trailing “/” (except root) # Strip duplicate slashes and trailing "/" (except root)
path = quote(unquote(parsed.path)) # IMPORTANT: Don't use quote(unquote()) as it mangles + signs in URLs
# The path from urlparse is already properly encoded
path = parsed.path
if path.endswith('/') and path != '/': if path.endswith('/') and path != '/':
path = path.rstrip('/') path = path.rstrip('/')