From 45d8e1450f11305c454a08f551404773c6dca213 Mon Sep 17 00:00:00 2001 From: unclecode Date: Sun, 15 Feb 2026 09:55:55 +0000 Subject: [PATCH] Fix proxy escalation: don't re-raise on first proxy exception when chain has alternatives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When proxy_config is a list (escalation chain) and the first proxy throws an exception (timeout, connection error, browser crash), the retry loop now continues to the next proxy instead of immediately re-raising. Previously, exceptions on _p_idx==0 and _attempt==0 were always re-raised, which broke the entire escalation chain — ISP/Residential/fallback proxies were never tried. This made the proxy list effectively useless for sites where the first-tier proxy fails with an exception rather than a blocked response. The raise is preserved when there's only a single proxy and single attempt (len(proxy_list) <= 1 and max_attempts <= 1) so that simple non-chain crawls still get immediate error propagation. --- crawl4ai/async_webcrawler.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/crawl4ai/async_webcrawler.py b/crawl4ai/async_webcrawler.py index 6470b645..9dbf98ef 100644 --- a/crawl4ai/async_webcrawler.py +++ b/crawl4ai/async_webcrawler.py @@ -513,15 +513,17 @@ class AsyncWebCrawler: "blocked": True, "reason": str(_crawl_err), }) - if _p_idx > 0 or _attempt > 0: - self.logger.error_status( - url=url, - error=f"Proxy {_proxy.server if _proxy else 'direct'} failed: {_crawl_err}", - tag="ANTIBOT", - ) - _block_reason = str(_crawl_err) - else: - raise # First attempt on first proxy propagates normally + self.logger.error_status( + url=url, + error=f"Proxy {_proxy.server if _proxy else 'direct'} failed: {_crawl_err}", + tag="ANTIBOT", + ) + _block_reason = str(_crawl_err) + # If this is the only proxy and only attempt, re-raise + # so the caller gets the real error (not a silent swallow). + # But if there are more proxies or retries to try, continue. + if len(_proxy_list) <= 1 and _max_attempts <= 1: + raise # Restore original proxy_config config.proxy_config = _original_proxy_config