Fix redirect target verification in AsyncUrlSeeder and enhance tests

- Added `verify_redirect_targets` parameter to control redirect verification.
- Modified `_resolve_head()` to verify redirect targets based on the new parameter.
- Implemented tests for both verification modes, ensuring dead redirects are filtered out and legacy behavior is preserved.

Dockerfile

@@ -167,11 +167,6 @@ RUN mkdir -p /home/appuser/.cache/ms-playwright \
 
 RUN crawl4ai-doctor
 
-# Ensure all cache directories belong to appuser
-# This fixes permission issues with .cache/url_seeder and other runtime cache dirs
-RUN mkdir -p /home/appuser/.cache \
-    && chown -R appuser:appuser /home/appuser/.cache
-
 # Copy application code
 COPY deploy/docker/* ${APP_HOME}/
 

crawl4ai/adaptive_crawler.py

@@ -728,18 +728,18 @@ class EmbeddingStrategy(CrawlStrategy):
         provider = llm_config_dict.get('provider', 'openai/gpt-4o-mini') if llm_config_dict else 'openai/gpt-4o-mini'
         api_token = llm_config_dict.get('api_token') if llm_config_dict else None
         
-        response = perform_completion_with_backoff(
-            provider=provider,
-            prompt_with_variables=prompt,
-            api_token=api_token,
-            json_response=True
-        )
+        # response = perform_completion_with_backoff(
+        #     provider=provider,
+        #     prompt_with_variables=prompt,
+        #     api_token=api_token,
+        #     json_response=True
+        # )
         
-        variations = json.loads(response.choices[0].message.content)
+        # variations = json.loads(response.choices[0].message.content)
         
         
         # # Mock data with more variations for split
-        # variations ={'queries': ['what are the best vegetables to use in fried rice?', 'how do I make vegetable fried rice from scratch?', 'can you provide a quick recipe for vegetable fried rice?', 'what cooking techniques are essential for perfect fried rice with vegetables?', 'how to add flavor to vegetable fried rice?', 'are there any tips for making healthy fried rice with vegetables?']}
+        variations ={'queries': ['what are the best vegetables to use in fried rice?', 'how do I make vegetable fried rice from scratch?', 'can you provide a quick recipe for vegetable fried rice?', 'what cooking techniques are essential for perfect fried rice with vegetables?', 'how to add flavor to vegetable fried rice?', 'are there any tips for making healthy fried rice with vegetables?']}
         
         
         # variations = {'queries': [

crawl4ai/async_configs.py

@@ -1792,10 +1792,7 @@ class LLMConfig:
         frequency_penalty: Optional[float] = None,
         presence_penalty: Optional[float] = None,
         stop: Optional[List[str]] = None,
-        n: Optional[int] = None,
-        backoff_base_delay: Optional[int] = None,
-        backoff_max_attempts: Optional[int] = None,
-        backoff_exponential_factor: Optional[int] = None,
+        n: Optional[int] = None,    
     ):
         """Configuaration class for LLM provider and API token."""
         self.provider = provider
@@ -1824,9 +1821,6 @@ class LLMConfig:
         self.presence_penalty = presence_penalty
         self.stop = stop
         self.n = n
-        self.backoff_base_delay = backoff_base_delay if backoff_base_delay is not None else 2
-        self.backoff_max_attempts = backoff_max_attempts if backoff_max_attempts is not None else 3
-        self.backoff_exponential_factor = backoff_exponential_factor if backoff_exponential_factor is not None else 2
 
     @staticmethod
     def from_kwargs(kwargs: dict) -> "LLMConfig":
@@ -1840,10 +1834,7 @@ class LLMConfig:
             frequency_penalty=kwargs.get("frequency_penalty"),
             presence_penalty=kwargs.get("presence_penalty"),
             stop=kwargs.get("stop"),
-            n=kwargs.get("n"),
-            backoff_base_delay=kwargs.get("backoff_base_delay"),
-            backoff_max_attempts=kwargs.get("backoff_max_attempts"),
-            backoff_exponential_factor=kwargs.get("backoff_exponential_factor")
+            n=kwargs.get("n")
         )
 
     def to_dict(self):
@@ -1857,10 +1848,7 @@ class LLMConfig:
             "frequency_penalty": self.frequency_penalty,
             "presence_penalty": self.presence_penalty,
             "stop": self.stop,
-            "n": self.n,
-            "backoff_base_delay": self.backoff_base_delay,
-            "backoff_max_attempts": self.backoff_max_attempts,
-            "backoff_exponential_factor": self.backoff_exponential_factor
+            "n": self.n
         }
 
     def clone(self, **kwargs):

crawl4ai/async_url_seeder.py

@@ -166,6 +166,22 @@ class AsyncUrlSeeder:
     Async version of UrlSeeder.
     Call pattern is await/async for / async with.
 
+    Parameters
+    ----------
+    ttl : timedelta, default TTL
+        Time-to-live for cached results.
+    client : httpx.AsyncClient, optional
+        HTTP client to use. If None, creates a new one.
+    logger : AsyncLoggerBase, optional
+        Logger instance for logging messages.
+    base_directory : str or pathlib.Path, optional
+        Base directory for cache storage. Defaults to home directory.
+    cache_root : str or pathlib.Path, optional
+        Root directory for URL seeder cache. Defaults to ~/.cache/url_seeder.
+    verify_redirect_targets : bool, default True
+        Whether to verify that redirect targets are alive (2xx status) before returning them.
+        When False, returns redirect targets without verification (legacy behavior).
+
     Public coroutines
     -----------------
     await seed.urls(...)
@@ -203,6 +219,8 @@ class AsyncUrlSeeder:
         # NEW: Add base_directory
         base_directory: Optional[Union[str, pathlib.Path]] = None,
         cache_root: Optional[Union[str, Path]] = None,
+        # NEW: Control redirect target verification
+        verify_redirect_targets: bool = True,
     ):
         self.ttl = ttl
         self._owns_client = client is None  # Track if we created the client
@@ -227,6 +245,9 @@ class AsyncUrlSeeder:
             cache_root or "~/.cache/url_seeder"))
         (self.cache_root / "live").mkdir(parents=True, exist_ok=True)
         (self.cache_root / "head").mkdir(exist_ok=True)
+        
+        # Store redirect verification setting
+        self.verify_redirect_targets = verify_redirect_targets
 
     def _log(self, level: str, message: str, tag: str = "URL_SEED", **kwargs: Any):
         """Helper to log messages using the provided logger, if available."""
@@ -682,24 +703,47 @@ class AsyncUrlSeeder:
 
         Returns:
             * the same URL if it answers 2xx,
-            * the absolute redirect target if it answers 3xx,
+            * the absolute redirect target if it answers 3xx (and if verify_redirect_targets=True, only if target is alive/2xx),
             * None on any other status or network error.
         """
         try:
             r = await self.client.head(url, timeout=10, follow_redirects=False)
-
-            # direct hit
+            # direct 2xx hit
             if 200 <= r.status_code < 300:
                 return str(r.url)
-
-            # single level redirect
+            # single-level redirect (3xx)
             if r.status_code in (301, 302, 303, 307, 308):
                 loc = r.headers.get("location")
                 if loc:
-                    return urljoin(url, loc)
-
+                    target = urljoin(url, loc)
+                    # Avoid infinite loop on self-redirect
+                    if target == url:
+                        return None
+                    
+                    # If not verifying redirect targets, return immediately (old behavior)
+                    if not self.verify_redirect_targets:
+                        return target
+                    
+                    # Verify redirect target is alive (new behavior)
+                    try:
+                        r2 = await self.client.head(target, timeout=10, follow_redirects=False)
+                        if 200 <= r2.status_code < 300:
+                            return str(r2.url)
+                        # Optionally, could handle another 3xx here for 2-step chains, but spec only says 1
+                        else:
+                            self._log(
+                                "debug",
+                                "HEAD redirect target {target} did not resolve: status {status}",
+                                params={"target": target, "status": r2.status_code},
+                                tag="URL_SEED",
+                            )
+                            return None
+                    except Exception as e2:
+                        self._log("debug", "HEAD {target} failed: {err}",
+                            params={"target": target, "err": str(e2)}, tag="URL_SEED")
+                        return None
+            # all other cases
             return None
-
         except Exception as e:
             self._log("debug", "HEAD {url} failed: {err}",
                       params={"url": url, "err": str(e)}, tag="URL_SEED")

crawl4ai/async_webcrawler.py

@@ -617,17 +617,17 @@ class AsyncWebCrawler:
                 else config.chunking_strategy
             )
             sections = chunking.chunk(content)
-            # extracted_content = config.extraction_strategy.run(_url, sections)
+            # extracted_content = config.extraction_strategy.run(url, sections)
 
             # Use async version if available for better parallelism
             if hasattr(config.extraction_strategy, 'arun'):
-                extracted_content = await config.extraction_strategy.arun(_url, sections)
+                extracted_content = await config.extraction_strategy.arun(url, sections)
             else:
                 # Fallback to sync version run in thread pool to avoid blocking
                 extracted_content = await asyncio.to_thread(
                     config.extraction_strategy.run, url, sections
                 )
-                
+
             extracted_content = json.dumps(
                 extracted_content, indent=4, default=str, ensure_ascii=False
             )

crawl4ai/content_filter_strategy.py

@@ -980,9 +980,6 @@ class LLMContentFilter(RelevantContentFilter):
                         prompt,
                         api_token,
                         base_url=base_url,
-                        base_delay=self.llm_config.backoff_base_delay,
-                        max_attempts=self.llm_config.backoff_max_attempts,
-                        exponential_factor=self.llm_config.backoff_exponential_factor,
                         extra_args=extra_args,
                     )
 

crawl4ai/extraction_strategy.py

@@ -649,9 +649,6 @@ class LLMExtractionStrategy(ExtractionStrategy):
                 base_url=self.llm_config.base_url,
                 json_response=self.force_json_response,
                 extra_args=self.extra_args,
-                base_delay=self.llm_config.backoff_base_delay,
-                max_attempts=self.llm_config.backoff_max_attempts,
-                exponential_factor=self.llm_config.backoff_exponential_factor
             )  # , json_response=self.extract_type == "schema")
             # Track usage
             usage = TokenUsage(
@@ -849,9 +846,6 @@ class LLMExtractionStrategy(ExtractionStrategy):
                 base_url=self.llm_config.base_url,
                 json_response=self.force_json_response,
                 extra_args=self.extra_args,
-                base_delay=self.llm_config.backoff_base_delay,
-                max_attempts=self.llm_config.backoff_max_attempts,
-                exponential_factor=self.llm_config.backoff_exponential_factor
             )
             # Track usage
             usage = TokenUsage(

crawl4ai/table_extraction.py

@@ -795,9 +795,6 @@ Return only a JSON array of extracted tables following the specified format."""
                     api_token=self.llm_config.api_token,
                     base_url=self.llm_config.base_url,
                     json_response=True,
-                    base_delay=self.llm_config.backoff_base_delay,
-                    max_attempts=self.llm_config.backoff_max_attempts,
-                    exponential_factor=self.llm_config.backoff_exponential_factor,
                     extra_args=self.extra_args
                 )
                 
@@ -1119,9 +1116,6 @@ Return only a JSON array of extracted tables following the specified format."""
                     api_token=self.llm_config.api_token,
                     base_url=self.llm_config.base_url,
                     json_response=True,
-                    base_delay=self.llm_config.backoff_base_delay,
-                    max_attempts=self.llm_config.backoff_max_attempts,
-                    exponential_factor=self.llm_config.backoff_exponential_factor,
                     extra_args=self.extra_args
                 )
                 

crawl4ai/utils.py

@@ -1745,9 +1745,6 @@ def perform_completion_with_backoff(
     api_token,
     json_response=False,
     base_url=None,
-    base_delay=2,
-    max_attempts=3,
-    exponential_factor=2,
     **kwargs,
 ):
     """
@@ -1764,9 +1761,6 @@ def perform_completion_with_backoff(
         api_token (str): The API token for authentication.
         json_response (bool): Whether to request a JSON response. Defaults to False.
         base_url (Optional[str]): The base URL for the API. Defaults to None.
-        base_delay (int): The base delay in seconds. Defaults to 2.
-        max_attempts (int): The maximum number of attempts. Defaults to 3.
-        exponential_factor (int): The exponential factor. Defaults to 2.
         **kwargs: Additional arguments for the API request.
 
     Returns:
@@ -1776,6 +1770,9 @@ def perform_completion_with_backoff(
     from litellm import completion
     from litellm.exceptions import RateLimitError
 
+    max_attempts = 3
+    base_delay = 2  # Base delay in seconds, you can adjust this based on your needs
+
     extra_args = {"temperature": 0.01, "api_key": api_token, "base_url": base_url}
     if json_response:
         extra_args["response_format"] = {"type": "json_object"}
@@ -1801,7 +1798,7 @@ def perform_completion_with_backoff(
             # Check if we have exhausted our max attempts
             if attempt < max_attempts - 1:
                 # Calculate the delay and wait
-                delay = base_delay * (exponential_factor**attempt)  # Exponential backoff formula
+                delay = base_delay * (2**attempt)  # Exponential backoff formula
                 print(f"Waiting for {delay} seconds before retrying...")
                 time.sleep(delay)
             else:
@@ -1834,9 +1831,6 @@ async def aperform_completion_with_backoff(
     api_token,
     json_response=False,
     base_url=None,
-    base_delay=2,
-    max_attempts=3,
-    exponential_factor=2,
     **kwargs,
 ):
     """
@@ -1853,9 +1847,6 @@ async def aperform_completion_with_backoff(
         api_token (str): The API token for authentication.
         json_response (bool): Whether to request a JSON response. Defaults to False.
         base_url (Optional[str]): The base URL for the API. Defaults to None.
-        base_delay (int): The base delay in seconds. Defaults to 2.
-        max_attempts (int): The maximum number of attempts. Defaults to 3.
-        exponential_factor (int): The exponential factor. Defaults to 2.
         **kwargs: Additional arguments for the API request.
 
     Returns:
@@ -1866,6 +1857,9 @@ async def aperform_completion_with_backoff(
     from litellm.exceptions import RateLimitError
     import asyncio
 
+    max_attempts = 3
+    base_delay = 2  # Base delay in seconds, you can adjust this based on your needs
+
     extra_args = {"temperature": 0.01, "api_key": api_token, "base_url": base_url}
     if json_response:
         extra_args["response_format"] = {"type": "json_object"}
@@ -1891,7 +1885,7 @@ async def aperform_completion_with_backoff(
             # Check if we have exhausted our max attempts
             if attempt < max_attempts - 1:
                 # Calculate the delay and wait
-                delay = base_delay * (exponential_factor**attempt)  # Exponential backoff formula
+                delay = base_delay * (2**attempt)  # Exponential backoff formula
                 print(f"Waiting for {delay} seconds before retrying...")
                 await asyncio.sleep(delay)
             else:

deploy/docker/api.py

@@ -108,10 +108,7 @@ async def handle_llm_qa(
             prompt_with_variables=prompt,
             api_token=get_llm_api_key(config),  # Returns None to let litellm handle it
             temperature=get_llm_temperature(config),
-            base_url=get_llm_base_url(config),
-            base_delay=config["llm"].get("backoff_base_delay", 2),
-            max_attempts=config["llm"].get("backoff_max_attempts", 3),
-            exponential_factor=config["llm"].get("backoff_exponential_factor", 2)
+            base_url=get_llm_base_url(config)
         )
 
         return response.choices[0].message.content

docs/md_v2/api/parameters.md

@@ -439,19 +439,10 @@ LLMConfig is useful to pass LLM provider config to strategies and functions that
 | **`provider`**    | `"ollama/llama3","groq/llama3-70b-8192","groq/llama3-8b-8192", "openai/gpt-4o-mini" ,"openai/gpt-4o","openai/o1-mini","openai/o1-preview","openai/o3-mini","openai/o3-mini-high","anthropic/claude-3-haiku-20240307","anthropic/claude-3-opus-20240229","anthropic/claude-3-sonnet-20240229","anthropic/claude-3-5-sonnet-20240620","gemini/gemini-pro","gemini/gemini-1.5-pro","gemini/gemini-2.0-flash","gemini/gemini-2.0-flash-exp","gemini/gemini-2.0-flash-lite-preview-02-05","deepseek/deepseek-chat"`<br/>*(default: `"openai/gpt-4o-mini"`)* | Which LLM provider to use. 
 | **`api_token`**         |1.Optional. When not provided explicitly, api_token will be read from environment variables based on provider. For example: If a gemini model is passed as provider then,`"GEMINI_API_KEY"` will be read from environment variables  <br/> 2. API token of LLM provider <br/> eg: `api_token = "gsk_1ClHGGJ7Lpn4WGybR7vNWGdyb3FY7zXEw3SCiy0BAVM9lL8CQv"` <br/> 3. Environment variable - use with prefix "env:" <br/> eg:`api_token = "env: GROQ_API_KEY"`              | API token to use for the given provider 
 | **`base_url`**         |Optional. Custom API endpoint | If your provider has a custom endpoint
-| **`backoff_base_delay`** |Optional. `int` *(default: `2`)* | Seconds to wait before the first retry when the provider throttles a request.
-| **`backoff_max_attempts`** |Optional. `int` *(default: `3`)* | Total tries (initial call + retries) before surfacing an error.
-| **`backoff_exponential_factor`** |Optional. `int` *(default: `2`)* | Multiplier that increases the wait time for each retry (`delay = base_delay * factor^attempt`).
 
 ## 3.2 Example Usage
 ```python
-llm_config = LLMConfig(
-    provider="openai/gpt-4o-mini",
-    api_token=os.getenv("OPENAI_API_KEY"),
-    backoff_base_delay=1, # optional
-    backoff_max_attempts=5, # optional
-    backoff_exponential_factor=3, # optional
-)
+llm_config = LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv("OPENAI_API_KEY"))
 ```
 
 ## 4. Putting It All Together

docs/md_v2/complete-sdk-reference.md

@@ -1593,20 +1593,8 @@ The `clone()` method:
     - Environment variable - use with prefix "env:" <br/> eg:`api_token = "env: GROQ_API_KEY"`            
 3. **`base_url`**:  
    - If your provider has a custom endpoint
-
-4. **Backoff controls** *(optional)*:  
-   - `backoff_base_delay` *(default `2` seconds)* – how long to pause before the first retry if the provider rate-limits you.  
-   - `backoff_max_attempts` *(default `3`)* – total tries for the same prompt (initial call + retries).  
-   - `backoff_exponential_factor` *(default `2`)* – how quickly the pause grows between retries. A factor of 2 yields waits like 2s → 4s → 8s.  
-   - Because these plug into Crawl4AI’s retry helper, every LLM strategy automatically follows the pacing you define here.
 ```python
-llm_config = LLMConfig(
-    provider="openai/gpt-4o-mini",
-    api_token=os.getenv("OPENAI_API_KEY"),
-    backoff_base_delay=1, # optional
-    backoff_max_attempts=5, # optional
-    backoff_exponential_factor=3, # optional
-)
+llm_config = LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv("OPENAI_API_KEY"))
 ```
 ## 4. Putting It All Together
 In a typical scenario, you define **one** `BrowserConfig` for your crawler session, then create **one or more** `CrawlerRunConfig` & `LLMConfig` depending on each call's needs:

docs/md_v2/core/browser-crawler-config.md

@@ -308,20 +308,8 @@ The `clone()` method:
 3.⠀**`base_url`**:  
    - If your provider has a custom endpoint
 
-4.⠀**Retry/backoff controls** *(optional)*:  
-   - `backoff_base_delay` *(default `2` seconds)* – base delay inserted before the first retry when the provider returns a rate-limit response.  
-   - `backoff_max_attempts` *(default `3`)* – total number of attempts (initial call plus retries) before the request is surfaced as an error.  
-   - `backoff_exponential_factor` *(default `2`)* – growth rate for the retry delay (`delay = base_delay * factor^attempt`).  
-   - These values are forwarded to the shared `perform_completion_with_backoff` helper, ensuring every strategy that consumes your `LLMConfig` honors the same throttling policy.
-
 ```python
-llm_config = LLMConfig(
-    provider="openai/gpt-4o-mini",
-    api_token=os.getenv("OPENAI_API_KEY"),
-    backoff_base_delay=1, # optional
-    backoff_max_attempts=5, # optional
-    backoff_exponential_factor=3, #optional
-)
+llm_config = LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv("OPENAI_API_KEY"))
 ```
 
 ## 4. Putting It All Together

tests/general/test_async_webcrawler.py

@@ -9,21 +9,6 @@ from crawl4ai import (
     RateLimiter,
     CacheMode
 )
-from crawl4ai.extraction_strategy import ExtractionStrategy
-
-class MockExtractionStrategy(ExtractionStrategy):
-    """Mock extraction strategy for testing URL parameter handling"""
-
-    def __init__(self):
-        super().__init__()
-        self.run_calls = []
-
-    def extract(self, url: str, html: str, *args, **kwargs):
-        return [{"test": "data"}]
-
-    def run(self, url: str, sections: List[str], *args, **kwargs):
-        self.run_calls.append(url)
-        return super().run(url, sections, *args, **kwargs)
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("viewport", [
@@ -157,72 +142,8 @@ async def test_error_handling(error_url):
         assert not result.success
         assert result.error_message is not None
 
-@pytest.mark.asyncio
-async def test_extraction_strategy_run_with_regular_url():
-    """
-    Regression test for extraction_strategy.run URL parameter handling with regular URLs.
-
-    This test verifies that when is_raw_html=False (regular URL),
-    extraction_strategy.run is called with the actual URL.
-    """
-    browser_config = BrowserConfig(
-        browser_type="chromium",
-        headless=True
-    )
-
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        mock_strategy = MockExtractionStrategy()
-
-        # Test regular URL (is_raw_html=False)
-        regular_url = "https://example.com"
-        result = await crawler.arun(
-            url=regular_url,
-            config=CrawlerRunConfig(
-                page_timeout=30000,
-                extraction_strategy=mock_strategy,
-                cache_mode=CacheMode.BYPASS
-            )
-        )
-
-        assert result.success
-        assert len(mock_strategy.run_calls) == 1
-        assert mock_strategy.run_calls[0] == regular_url, f"Expected '{regular_url}', got '{mock_strategy.run_calls[0]}'"
-
-@pytest.mark.asyncio
-async def test_extraction_strategy_run_with_raw_html():
-    """
-    Regression test for extraction_strategy.run URL parameter handling with raw HTML.
-
-    This test verifies that when is_raw_html=True (URL starts with "raw:"),
-    extraction_strategy.run is called with "Raw HTML" instead of the actual URL.
-    """
-    browser_config = BrowserConfig(
-        browser_type="chromium",
-        headless=True
-    )
-
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        mock_strategy = MockExtractionStrategy()
-
-        # Test raw HTML URL (is_raw_html=True automatically set)
-        raw_html_url = "raw:<html><body><h1>Test HTML</h1><p>This is a test.</p></body></html>"
-        result = await crawler.arun(
-            url=raw_html_url,
-            config=CrawlerRunConfig(
-                page_timeout=30000,
-                extraction_strategy=mock_strategy,
-                cache_mode=CacheMode.BYPASS
-            )
-        )
-
-        assert result.success
-        assert len(mock_strategy.run_calls) == 1
-        assert mock_strategy.run_calls[0] == "Raw HTML", f"Expected 'Raw HTML', got '{mock_strategy.run_calls[0]}'"
-
 if __name__ == "__main__":
     asyncio.run(test_viewport_config((1024, 768)))
     asyncio.run(test_memory_management())
     asyncio.run(test_rate_limiting())
-    asyncio.run(test_javascript_execution())
-    asyncio.run(test_extraction_strategy_run_with_regular_url())
-    asyncio.run(test_extraction_strategy_run_with_raw_html())
+    asyncio.run(test_javascript_execution())

tests/test_async_url_seeder.py

@@ -0,0 +1,29 @@
+import pytest
+import asyncio
+from crawl4ai.async_url_seeder import AsyncUrlSeeder
+
+@pytest.mark.asyncio
+async def test_resolve_head_handles_dead_redirects():
+    seeder = AsyncUrlSeeder()
+    # Should return None – redirects to a dead URL
+    assert await seeder._resolve_head("http://youtube.com/sitemap.xml") is None
+    assert await seeder._resolve_head("https://stripe.com/sitemap.xml") is None
+
+@pytest.mark.asyncio
+async def test_resolve_head_direct_hit():
+    seeder = AsyncUrlSeeder()
+    # Test with a known live URL, e.g., httpbin
+    result = await seeder._resolve_head("https://httpbin.org/status/200")
+    assert result == "https://httpbin.org/status/200"
+
+@pytest.mark.asyncio
+async def test_resolve_head_verify_redirect_targets_false():
+    # Test with verification disabled - should return redirect target without checking if alive
+    seeder = AsyncUrlSeeder(verify_redirect_targets=False)
+    # This should return the redirect target even if it's dead (old behavior)
+    result = await seeder._resolve_head("http://youtube.com/sitemap.xml")
+    # The exact redirect target might vary, but it should not be None
+    assert result is not None
+    assert isinstance(result, str)
+    # Should be different from the input URL (indicating redirect was followed)
+    assert result != "http://youtube.com/sitemap.xml"