Make proxy_config a property so direct assignment also normalizes

Setting config.proxy_config = [ProxyConfig.DIRECT, ...] after
construction now goes through the same normalization as __init__,
converting "direct" sentinels to None. Fixes crash when proxy_config
is assigned directly instead of passed to the constructor.
This commit is contained in:
unclecode
2026-02-14 13:16:36 +00:00
parent 879553955c
commit d028a889d0

View File

@@ -1496,25 +1496,7 @@ class CrawlerRunConfig():
self.prettiify = prettiify self.prettiify = prettiify
self.parser_type = parser_type self.parser_type = parser_type
self.scraping_strategy = scraping_strategy or LXMLWebScrapingStrategy() self.scraping_strategy = scraping_strategy or LXMLWebScrapingStrategy()
# Normalize proxy_config: single ProxyConfig stored as-is, list stored as list self.proxy_config = proxy_config # runs through property setter
if isinstance(proxy_config, list):
normalized = []
for p in proxy_config:
if p is None or p == "direct":
normalized.append(None)
elif isinstance(p, dict):
normalized.append(ProxyConfig.from_dict(p))
elif isinstance(p, str):
normalized.append(ProxyConfig.from_string(p))
else:
normalized.append(p)
self.proxy_config = normalized
elif isinstance(proxy_config, dict):
self.proxy_config = ProxyConfig.from_dict(proxy_config)
elif isinstance(proxy_config, str):
self.proxy_config = ProxyConfig.from_string(proxy_config)
else:
self.proxy_config = proxy_config # ProxyConfig or None
self.proxy_rotation_strategy = proxy_rotation_strategy self.proxy_rotation_strategy = proxy_rotation_strategy
@@ -1681,6 +1663,37 @@ class CrawlerRunConfig():
self._compile_c4a_script() self._compile_c4a_script()
@staticmethod
def _normalize_proxy_config(value):
"""Normalize proxy_config to ProxyConfig, list of ProxyConfig/None, or None."""
if isinstance(value, list):
normalized = []
for p in value:
if p is None or p == "direct":
normalized.append(None)
elif isinstance(p, dict):
normalized.append(ProxyConfig.from_dict(p))
elif isinstance(p, str):
normalized.append(ProxyConfig.from_string(p))
else:
normalized.append(p)
return normalized
elif isinstance(value, dict):
return ProxyConfig.from_dict(value)
elif isinstance(value, str):
if value == "direct":
return None
return ProxyConfig.from_string(value)
return value # ProxyConfig or None
@property
def proxy_config(self):
return self._proxy_config
@proxy_config.setter
def proxy_config(self, value):
self._proxy_config = CrawlerRunConfig._normalize_proxy_config(value)
def _get_proxy_list(self) -> list: def _get_proxy_list(self) -> list:
"""Normalize proxy_config to a list for the retry loop.""" """Normalize proxy_config to a list for the retry loop."""
if self.proxy_config is None: if self.proxy_config is None: