ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge PR #1364: Fix script tag removal losing adjacent text in cleaned_html

Browse Source
This commit is contained in:
unclecode
2026-02-01 06:22:10 +00:00
parent 0f83b05a2d 660d7011b9
commit a244e4d781
1 changed files with 21 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
crawl4ai/content_scraping_strategy.py
View File

@@ -709,11 +709,31 @@ class LXMLWebScrapingStrategy(ContentScrapingStrategy):
content_element = body
# Remove script and style tags
for tag in ["script", "style", "link", "meta", "noscript"]:
for tag in ["style", "link", "meta", "noscript"]:
for element in body.xpath(f".//{tag}"):
if element.getparent() is not None:
element.getparent().remove(element)
# Handle script separately
for element in body.xpath(f".//script"):
parent = element.getparent()
if parent is not None:
tail = element.tail # Get the tail text
if tail:
prev = element.getprevious() # Get the previous sibling node
if prev is not None:
if prev.tail:
prev.tail += tail
else:
prev.tail = tail
else:
if parent.text:
parent.text += tail
else:
parent.text = tail
parent.remove(element) # Delete the element
# Handle social media and domain exclusions
kwargs["exclude_domains"] = set(kwargs.get("exclude_domains", []))
if kwargs.get("exclude_social_media_links", False):