ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Enable following redirects in sitemap fetching for seeder

Browse Source
This commit is contained in:
Charlie C
2025-07-25 15:57:09 +08:00
parent e3281935bc
commit 508b6fc233
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crawl4ai/async_url_seeder.py
View File

@@ -829,7 +829,7 @@ class AsyncUrlSeeder:
async def _iter_sitemap(self, url: str): async def _iter_sitemap(self, url: str):
try: try:
r = await self.client.get(url, timeout=15) r = await self.client.get(url, timeout=15, follow_redirects=True)
r.raise_for_status() r.raise_for_status()
except httpx.HTTPStatusError as e: except httpx.HTTPStatusError as e:
self._log("warning", "Failed to fetch sitemap {url}: HTTP {status_code}", self._log("warning", "Failed to fetch sitemap {url}: HTTP {status_code}",