ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Implement base tag support in link extraction (#1147)

Browse Source 
- Extract base href from <head><base> tag using XPath in _process_element method
- Use base URL as the primary URL for link normalization when present
- Add error handling with logging for malformed or problematic base tags
- Maintain backward compatibility when no base tag is present
- Add test to verify the functionality of the base tag extraction.
This commit is contained in:
Soham Kukreti
2025-08-08 20:00:11 +05:30
parent a5bcac4c9d
commit 18ad3ef159
2 changed files with 21 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
crawl4ai/content_scraping_strategy.py
View File

@@ -242,6 +242,16 @@ class LXMLWebScrapingStrategy(ContentScrapingStrategy):
exclude_domains = set(kwargs.get("exclude_domains", []))
# Process links
try:
base_element = element.xpath("//head/base[@href]")
if base_element:
base_href = base_element[0].get("href", "").strip()
if base_href:
url = base_href
except Exception as e:
self._log("error", f"Error extracting base URL: {str(e)}", "SCRAPE")
pass
for link in element.xpath(".//a[@href]"):
href = link.get("href", "").strip()
if not href:

11
tests/async/test_content_extraction.py
View File

@@ -91,6 +91,17 @@ async def test_css_selector_extraction():
assert result.markdown
assert all(heading in result.markdown for heading in ["#", "##", "###"])
@pytest.mark.asyncio
async def test_base_tag_link_extraction():
async with AsyncWebCrawler(verbose=True) as crawler:
url = "https://sohamkukreti.github.io/portfolio"
result = await crawler.arun(url=url)
assert result.success
assert result.links
assert isinstance(result.links, dict)
assert "internal" in result.links
assert "external" in result.links
assert any("github.com" in x["href"] for x in result.links["external"])
# Entry point for debugging
if __name__ == "__main__":