fix: Enable following redirects in sitemap fetching for seeder

2025-07-25 15:57:09 +08:00
parent e3281935bc
commit 508b6fc233
1 changed files with 1 additions and 1 deletions
--- a/crawl4ai/async_url_seeder.py
+++ b/crawl4ai/async_url_seeder.py
@@ -829,7 +829,7 @@ class AsyncUrlSeeder:

    async def _iter_sitemap(self, url: str):
        try:
-            r = await self.client.get(url, timeout=15)
+            r = await self.client.get(url, timeout=15, follow_redirects=True)
            r.raise_for_status()
        except httpx.HTTPStatusError as e:
            self._log("warning", "Failed to fetch sitemap {url}: HTTP {status_code}",