diff --git a/crawl4ai/async_url_seeder.py b/crawl4ai/async_url_seeder.py index 57ad5186..d2564797 100644 --- a/crawl4ai/async_url_seeder.py +++ b/crawl4ai/async_url_seeder.py @@ -829,7 +829,7 @@ class AsyncUrlSeeder: async def _iter_sitemap(self, url: str): try: - r = await self.client.get(url, timeout=15) + r = await self.client.get(url, timeout=15, follow_redirects=True) r.raise_for_status() except httpx.HTTPStatusError as e: self._log("warning", "Failed to fetch sitemap {url}: HTTP {status_code}",