#1559 :Add tests for sitemap parsing and URL normalization in AsyncUrlSeeder

crawl4ai/async_url_seeder.py

@@ -845,6 +845,15 @@ class AsyncUrlSeeder:
             return
 
         data = gzip.decompress(r.content) if url.endswith(".gz") else r.content
+        base_url = str(r.url)
+
+        def _normalize_loc(raw: Optional[str]) -> Optional[str]:
+            if not raw:
+                return None
+            normalized = urljoin(base_url, raw.strip())
+            if not normalized:
+                return None
+            return normalized
 
         # Detect if this is a sitemap index by checking for <sitemapindex> or presence of <sitemap> elements
         is_sitemap_index = False
@@ -857,25 +866,42 @@ class AsyncUrlSeeder:
                 # Use XML parser for sitemaps, not HTML parser
                 parser = etree.XMLParser(recover=True)
                 root = etree.fromstring(data, parser=parser)
+                # Namespace-agnostic lookups using local-name() so we honor custom or missing namespaces
+                sitemap_loc_nodes = root.xpath("//*[local-name()='sitemap']/*[local-name()='loc']")
+                url_loc_nodes = root.xpath("//*[local-name()='url']/*[local-name()='loc']")
 
-                # Define namespace for sitemap
+                self._log(
-                ns = {'s': 'http://www.sitemaps.org/schemas/sitemap/0.9'}
+                    "debug",
+                    "Parsed sitemap {url}: {sitemap_count} sitemap entries, {url_count} url entries discovered",
+                    params={
+                        "url": url,
+                        "sitemap_count": len(sitemap_loc_nodes),
+                        "url_count": len(url_loc_nodes),
+                    },
+                    tag="URL_SEED",
+                )
 
                 # Check for sitemap index entries
-                sitemap_locs = root.xpath('//s:sitemap/s:loc', namespaces=ns)
+                if sitemap_loc_nodes:
-                if sitemap_locs:
                     is_sitemap_index = True
-                    for sitemap_elem in sitemap_locs:
+                    for sitemap_elem in sitemap_loc_nodes:
-                        loc = sitemap_elem.text.strip() if sitemap_elem.text else ""
+                        loc = _normalize_loc(sitemap_elem.text)
                         if loc:
                             sub_sitemaps.append(loc)
 
                 # If not a sitemap index, get regular URLs
                 if not is_sitemap_index:
-                    for loc_elem in root.xpath('//s:url/s:loc', namespaces=ns):
+                    for loc_elem in url_loc_nodes:
-                        loc = loc_elem.text.strip() if loc_elem.text else ""
+                        loc = _normalize_loc(loc_elem.text)
                         if loc:
                             regular_urls.append(loc)
+                    if not regular_urls:
+                        self._log(
+                            "warning",
+                            "No <loc> entries found inside <url> tags for sitemap {url}. The sitemap might be empty or use an unexpected structure.",
+                            params={"url": url},
+                            tag="URL_SEED",
+                        )
             except Exception as e:
                 self._log("error", "LXML parsing error for sitemap {url}: {error}",
                           params={"url": url, "error": str(e)}, tag="URL_SEED")
@@ -892,19 +918,39 @@ class AsyncUrlSeeder:
 
                 # Check for sitemap index entries
                 sitemaps = root.findall('.//sitemap')
+                url_entries = root.findall('.//url')
+                self._log(
+                    "debug",
+                    "ElementTree parsed sitemap {url}: {sitemap_count} sitemap entries, {url_count} url entries discovered",
+                    params={
+                        "url": url,
+                        "sitemap_count": len(sitemaps),
+                        "url_count": len(url_entries),
+                    },
+                    tag="URL_SEED",
+                )
                 if sitemaps:
                     is_sitemap_index = True
                     for sitemap in sitemaps:
                         loc_elem = sitemap.find('loc')
-                        if loc_elem is not None and loc_elem.text:
+                        loc = _normalize_loc(loc_elem.text if loc_elem is not None else None)
-                            sub_sitemaps.append(loc_elem.text.strip())
+                        if loc:
+                            sub_sitemaps.append(loc)
 
                 # If not a sitemap index, get regular URLs
                 if not is_sitemap_index:
-                    for url_elem in root.findall('.//url'):
+                    for url_elem in url_entries:
                         loc_elem = url_elem.find('loc')
-                        if loc_elem is not None and loc_elem.text:
+                        loc = _normalize_loc(loc_elem.text if loc_elem is not None else None)
-                            regular_urls.append(loc_elem.text.strip())
+                        if loc:
+                            regular_urls.append(loc)
+                    if not regular_urls:
+                        self._log(
+                            "warning",
+                            "No <loc> entries found inside <url> tags for sitemap {url}. The sitemap might be empty or use an unexpected structure.",
+                            params={"url": url},
+                            tag="URL_SEED",
+                        )
             except Exception as e:
                 self._log("error", "ElementTree parsing error for sitemap {url}: {error}",
                           params={"url": url, "error": str(e)}, tag="URL_SEED")

docs/md_v2/advanced/proxy-security.md

@@ -1,304 +1,98 @@
-# Proxy & Security
+# Proxy 
-
-This guide covers proxy configuration and security features in Crawl4AI, including SSL certificate analysis and proxy rotation strategies.
-
-## Understanding Proxy Configuration
-
-Crawl4AI recommends configuring proxies per request through `CrawlerRunConfig.proxy_config`. This gives you precise control, enables rotation strategies, and keeps examples simple enough to copy, paste, and run.
 
 ## Basic Proxy Setup
 
-Configure proxies that apply to each crawl operation:
+Simple proxy configuration with `BrowserConfig`:
 
 ```python
-import asyncio
+from crawl4ai.async_configs import BrowserConfig
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, ProxyConfig
 
-run_config = CrawlerRunConfig(proxy_config=ProxyConfig(server="http://proxy.example.com:8080"))
+# Using HTTP proxy
-# run_config = CrawlerRunConfig(proxy_config={"server": "http://proxy.example.com:8080"})
+browser_config = BrowserConfig(proxy_config={"server": "http://proxy.example.com:8080"})
-# run_config = CrawlerRunConfig(proxy_config="http://proxy.example.com:8080")
+async with AsyncWebCrawler(config=browser_config) as crawler:
+    result = await crawler.arun(url="https://example.com")
 
-
+# Using SOCKS proxy
-async def main():
+browser_config = BrowserConfig(proxy_config={"server": "socks5://proxy.example.com:1080"})
-    browser_config = BrowserConfig()
+async with AsyncWebCrawler(config=browser_config) as crawler:
-    async with AsyncWebCrawler(config=browser_config) as crawler:
+    result = await crawler.arun(url="https://example.com")
-        result = await crawler.arun(url="https://example.com", config=run_config)
-        print(f"Success: {result.success} -> {result.url}")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
 ```
 
-!!! note "Why request-level?"
+## Authenticated Proxy
-    `CrawlerRunConfig.proxy_config` keeps each request self-contained, so swapping proxies or rotation strategies is just a matter of building a new run configuration.
 
-## Supported Proxy Formats
+Use an authenticated proxy with `BrowserConfig`:
-
-The `ProxyConfig.from_string()` method supports multiple formats:
 
 ```python
-from crawl4ai import ProxyConfig
+from crawl4ai.async_configs import BrowserConfig
 
-# HTTP proxy with authentication
+browser_config = BrowserConfig(proxy_config={
-proxy1 = ProxyConfig.from_string("http://user:pass@192.168.1.1:8080")
+    "server": "http://[host]:[port]",
-
+    "username": "[username]",
-# HTTPS proxy
+    "password": "[password]",
-proxy2 = ProxyConfig.from_string("https://proxy.example.com:8080")
+})
-
+async with AsyncWebCrawler(config=browser_config) as crawler:
-# SOCKS5 proxy
+    result = await crawler.arun(url="https://example.com")
-proxy3 = ProxyConfig.from_string("socks5://proxy.example.com:1080")
-
-# Simple IP:port format
-proxy4 = ProxyConfig.from_string("192.168.1.1:8080")
-
-# IP:port:user:pass format
-proxy5 = ProxyConfig.from_string("192.168.1.1:8080:user:pass")
 ```
 
-## Authenticated Proxies
 
-For proxies requiring authentication:
+## Rotating Proxies 
+
+Example using a proxy rotation service dynamically:
 
 ```python
-import asyncio
-from crawl4ai import AsyncWebCrawler,BrowserConfig, CrawlerRunConfig, ProxyConfig
-
-run_config = CrawlerRunConfig(
-    proxy_config=ProxyConfig(
-        server="http://proxy.example.com:8080",
-        username="your_username",
-        password="your_password",
-    )
-)
-# Or dictionary style:
-# run_config = CrawlerRunConfig(proxy_config={
-#     "server": "http://proxy.example.com:8080",
-#     "username": "your_username",
-#     "password": "your_password",
-# })
-
-
-async def main():
-    browser_config = BrowserConfig()
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        result = await crawler.arun(url="https://example.com", config=run_config)
-        print(f"Success: {result.success} -> {result.url}")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
-```
-
-## Environment Variable Configuration
-
-Load proxies from environment variables for easy configuration:
-
-```python
-import os
-from crawl4ai import ProxyConfig, CrawlerRunConfig
-
-# Set environment variable
-os.environ["PROXIES"] = "ip1:port1:user1:pass1,ip2:port2:user2:pass2,ip3:port3"
-
-# Load all proxies
-proxies = ProxyConfig.from_env()
-print(f"Loaded {len(proxies)} proxies")
-
-# Use first proxy
-if proxies:
-    run_config = CrawlerRunConfig(proxy_config=proxies[0])
-```
-
-## Rotating Proxies
-
-Crawl4AI supports automatic proxy rotation to distribute requests across multiple proxy servers. Rotation is applied per request using a rotation strategy on `CrawlerRunConfig`.
-
-### Proxy Rotation (recommended)
-```python
-import asyncio
 import re
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, ProxyConfig
+from crawl4ai import (
-from crawl4ai.proxy_strategy import RoundRobinProxyStrategy
+    AsyncWebCrawler,
-
+    BrowserConfig,
+    CrawlerRunConfig,
+    CacheMode,
+    RoundRobinProxyStrategy,
+)
+import asyncio
+from crawl4ai import ProxyConfig
 async def main():
-    # Load proxies from environment
+    # Load proxies and create rotation strategy
     proxies = ProxyConfig.from_env()
+    #eg: export PROXIES="ip1:port1:username1:password1,ip2:port2:username2:password2"
     if not proxies:
-        print("No proxies found! Set PROXIES environment variable.")
+        print("No proxies found in environment. Set PROXIES env variable!")
         return
 
-    # Create rotation strategy
     proxy_strategy = RoundRobinProxyStrategy(proxies)
 
-    # Configure per-request with proxy rotation
+    # Create configs
     browser_config = BrowserConfig(headless=True, verbose=False)
     run_config = CrawlerRunConfig(
         cache_mode=CacheMode.BYPASS,
-        proxy_rotation_strategy=proxy_strategy,
+        proxy_rotation_strategy=proxy_strategy
     )
 
     async with AsyncWebCrawler(config=browser_config) as crawler:
         urls = ["https://httpbin.org/ip"] * (len(proxies) * 2)  # Test each proxy twice
 
-        print(f"🚀 Testing {len(proxies)} proxies with rotation...")
+        print("\n📈 Initializing crawler with proxy rotation...")
-        results = await crawler.arun_many(urls=urls, config=run_config)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            print("\n🚀 Starting batch crawl with proxy rotation...")
+            results = await crawler.arun_many(
+                urls=urls,
+                config=run_config
+            )
+            for result in results:
+                if result.success:
+                    ip_match = re.search(r'(?:[0-9]{1,3}\.){3}[0-9]{1,3}', result.html)
+                    current_proxy = run_config.proxy_config if run_config.proxy_config else None
 
-        for i, result in enumerate(results):
+                    if current_proxy and ip_match:
-            if result.success:
+                        print(f"URL {result.url}")
-                # Extract IP from response
+                        print(f"Proxy {current_proxy.server} -> Response IP: {ip_match.group(0)}")
-                ip_match = re.search(r'(?:[0-9]{1,3}\.){3}[0-9]{1,3}', result.html)
+                        verified = ip_match.group(0) == current_proxy.ip
-                if ip_match:
+                        if verified:
-                    detected_ip = ip_match.group(0)
+                            print(f"✅ Proxy working! IP matches: {current_proxy.ip}")
-                    proxy_index = i % len(proxies)
+                        else:
-                    expected_ip = proxies[proxy_index].ip
+                            print("❌ Proxy failed or IP mismatch!")
+                    print("---")
 
-                    print(f"✅ Request {i+1}: Proxy {proxy_index+1} -> IP {detected_ip}")
+asyncio.run(main())
-                    if detected_ip == expected_ip:
-                        print("   🎯 IP matches proxy configuration")
-                    else:
-                        print(f"   ⚠️  IP mismatch (expected {expected_ip})")
-                else:
-                    print(f"❌ Request {i+1}: Could not extract IP from response")
-            else:
-                print(f"❌ Request {i+1}: Failed - {result.error_message}")
 
-if __name__ == "__main__":
-    asyncio.run(main())
 ```
 
-## SSL Certificate Analysis
-
-Combine proxy usage with SSL certificate inspection for enhanced security analysis. SSL certificate fetching is configured per request via `CrawlerRunConfig`.
-
-### Per-Request SSL Certificate Analysis
-```python
-import asyncio
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
-
-run_config = CrawlerRunConfig(
-    proxy_config={
-        "server": "http://proxy.example.com:8080",
-        "username": "user",
-        "password": "pass",
-    },
-    fetch_ssl_certificate=True,  # Enable SSL certificate analysis for this request
-)
-
-
-async def main():
-    browser_config = BrowserConfig()
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        result = await crawler.arun(url="https://example.com", config=run_config)
-
-        if result.success:
-            print(f"✅ Crawled via proxy: {result.url}")
-
-            # Analyze SSL certificate
-            if result.ssl_certificate:
-                cert = result.ssl_certificate
-                print("🔒 SSL Certificate Info:")
-                print(f"   Issuer: {cert.issuer}")
-                print(f"   Subject: {cert.subject}")
-                print(f"   Valid until: {cert.valid_until}")
-                print(f"   Fingerprint: {cert.fingerprint}")
-
-                # Export certificate
-                cert.to_json("certificate.json")
-                print("💾 Certificate exported to certificate.json")
-            else:
-                print("⚠️  No SSL certificate information available")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
-```
-
-## Security Best Practices
-
-### 1. Proxy Rotation for Anonymity
-```python
-from crawl4ai import CrawlerRunConfig, ProxyConfig
-from crawl4ai.proxy_strategy import RoundRobinProxyStrategy
-
-# Use multiple proxies to avoid IP blocking
-proxies = ProxyConfig.from_env("PROXIES")
-strategy = RoundRobinProxyStrategy(proxies)
-
-# Configure rotation per request (recommended)
-run_config = CrawlerRunConfig(proxy_rotation_strategy=strategy)
-
-# For a fixed proxy across all requests, just reuse the same run_config instance
-static_run_config = run_config
-```
-
-### 2. SSL Certificate Verification
-```python
-from crawl4ai import CrawlerRunConfig
-
-# Always verify SSL certificates when possible
-# Per-request (affects specific requests)
-run_config = CrawlerRunConfig(fetch_ssl_certificate=True)
-```
-
-### 3. Environment Variable Security
-```bash
-# Use environment variables for sensitive proxy credentials
-# Avoid hardcoding usernames/passwords in code
-export PROXIES="ip1:port1:user1:pass1,ip2:port2:user2:pass2"
-```
-
-### 4. SOCKS5 for Enhanced Security
-```python
-from crawl4ai import CrawlerRunConfig
-
-# Prefer SOCKS5 proxies for better protocol support
-run_config = CrawlerRunConfig(proxy_config="socks5://proxy.example.com:1080")
-```
-
-## Migration from Deprecated `proxy` Parameter
-
-!!! warning "Deprecation Notice"
-    The legacy `proxy` argument on `BrowserConfig` is deprecated. Configure proxies through `CrawlerRunConfig.proxy_config` so each request fully describes its network settings.
-
-```python
-# Old (deprecated) approach
-# from crawl4ai import BrowserConfig
-# browser_config = BrowserConfig(proxy="http://proxy.example.com:8080")
-
-# New (preferred) approach
-from crawl4ai import CrawlerRunConfig
-run_config = CrawlerRunConfig(proxy_config="http://proxy.example.com:8080")
-```
-
-### Safe Logging of Proxies
-```python
-from crawl4ai import ProxyConfig
-
-def safe_proxy_repr(proxy: ProxyConfig):
-    if getattr(proxy, "username", None):
-        return f"{proxy.server} (auth: ****)"
-    return proxy.server
-```
-
-## Troubleshooting
-
-### Common Issues
-
-???+ question "Proxy connection failed"
-    - Verify the proxy server is reachable from your network.
-    - Double-check authentication credentials.
-    - Ensure the protocol matches (`http`, `https`, or `socks5`).
-
-???+ question "SSL certificate errors"
-    - Some proxies break SSL inspection; switch proxies if you see repeated failures.
-    - Consider temporarily disabling certificate fetching to isolate the issue.
-
-???+ question "Environment variables not loading"
-    - Confirm `PROXIES` (or your custom env var) is set before running the script.
-    - Check formatting: `ip:port:user:pass,ip:port:user:pass`.
-
-???+ question "Proxy rotation not working"
-    - Ensure `ProxyConfig.from_env()` actually loaded entries (`len(proxies) > 0`).
-    - Attach `proxy_rotation_strategy` to `CrawlerRunConfig`.
-    - Validate the proxy definitions you pass into the strategy.

tests/unit/test_sitemap_namespace_parsing.py

@@ -0,0 +1,134 @@
+import sys
+from types import SimpleNamespace
+
+import pytest
+
+# Provide a lightweight stub for rank_bm25 before importing the seeder to avoid
+# optional dependency issues (e.g., incompatible wheels in CI).
+class _FakeBM25:
+    def __init__(self, corpus):
+        self._scores = [1.0] * len(corpus)
+
+    def get_scores(self, tokens):
+        return self._scores
+
+
+sys.modules.setdefault("rank_bm25", SimpleNamespace(BM25Okapi=_FakeBM25))
+
+from crawl4ai.async_url_seeder import AsyncUrlSeeder
+
+
+class DummyResponse:
+    def __init__(self, request_url: str, text: str):
+        self.status_code = 200
+        self._content = text.encode("utf-8")
+        self.url = request_url
+
+    def raise_for_status(self):
+        return None
+
+    @property
+    def content(self):
+        return self._content
+
+    @property
+    def text(self):
+        return self._content.decode("utf-8")
+
+
+class DummyAsyncClient:
+    def __init__(self, response_map):
+        self._responses = response_map
+
+    async def get(self, url, **kwargs):
+        payload = self._responses[url]
+        if callable(payload):
+            payload = payload()
+        return DummyResponse(url, payload)
+
+
+@pytest.mark.asyncio
+async def test_iter_sitemap_handles_namespace_less_sitemaps():
+    xml = """<?xml version="1.0"?>
+    <urlset>
+        <url><loc>https://example.com/a</loc></url>
+        <url><loc>https://example.com/b</loc></url>
+    </urlset>
+    """
+    seeder = AsyncUrlSeeder(client=DummyAsyncClient({"https://example.com/sitemap.xml": xml}))
+
+    urls = []
+    async for u in seeder._iter_sitemap("https://example.com/sitemap.xml"):
+        urls.append(u)
+
+    assert urls == ["https://example.com/a", "https://example.com/b"]
+
+
+@pytest.mark.asyncio
+async def test_iter_sitemap_handles_custom_namespace():
+    xml = """<?xml version="1.0"?>
+    <urlset xmlns="https://custom.namespace/schema">
+        <url><loc>https://example.com/ns</loc></url>
+    </urlset>
+    """
+    seeder = AsyncUrlSeeder(client=DummyAsyncClient({"https://example.com/ns-sitemap.xml": xml}))
+
+    urls = []
+    async for u in seeder._iter_sitemap("https://example.com/ns-sitemap.xml"):
+        urls.append(u)
+
+    assert urls == ["https://example.com/ns"]
+
+
+@pytest.mark.asyncio
+async def test_iter_sitemap_handles_namespace_index_and_children():
+    index_xml = """<?xml version="1.0"?>
+    <sitemapindex xmlns="http://another.example/ns">
+        <sitemap>
+            <loc>https://example.com/child-1.xml</loc>
+        </sitemap>
+        <sitemap>
+            <loc>https://example.com/child-2.xml</loc>
+        </sitemap>
+    </sitemapindex>
+    """
+    child_xml = """<?xml version="1.0"?>
+    <urlset xmlns="http://irrelevant">
+        <url><loc>https://example.com/page-{n}</loc></url>
+    </urlset>
+    """
+    responses = {
+        "https://example.com/index.xml": index_xml,
+        "https://example.com/child-1.xml": child_xml.format(n=1),
+        "https://example.com/child-2.xml": child_xml.format(n=2),
+    }
+    seeder = AsyncUrlSeeder(client=DummyAsyncClient(responses))
+
+    urls = []
+    async for u in seeder._iter_sitemap("https://example.com/index.xml"):
+        urls.append(u)
+
+    assert sorted(urls) == [
+        "https://example.com/page-1",
+        "https://example.com/page-2",
+    ]
+
+
+@pytest.mark.asyncio
+async def test_iter_sitemap_normalizes_relative_locations():
+    xml = """<?xml version="1.0"?>
+    <urlset>
+        <url><loc>/relative-path</loc></url>
+        <url><loc>https://example.com/absolute</loc></url>
+    </urlset>
+    """
+    seeder = AsyncUrlSeeder(client=DummyAsyncClient({"https://example.com/sitemap.xml": xml}))
+
+    urls = []
+    async for u in seeder._iter_sitemap("https://example.com/sitemap.xml"):
+        urls.append(u)
+
+    assert urls == [
+        "https://example.com/relative-path",
+        "https://example.com/absolute",
+    ]