Make proxy_config a property so direct assignment also normalizes
Setting config.proxy_config = [ProxyConfig.DIRECT, ...] after construction now goes through the same normalization as __init__, converting "direct" sentinels to None. Fixes crash when proxy_config is assigned directly instead of passed to the constructor.
This commit is contained in:
@@ -1496,25 +1496,7 @@ class CrawlerRunConfig():
|
|||||||
self.prettiify = prettiify
|
self.prettiify = prettiify
|
||||||
self.parser_type = parser_type
|
self.parser_type = parser_type
|
||||||
self.scraping_strategy = scraping_strategy or LXMLWebScrapingStrategy()
|
self.scraping_strategy = scraping_strategy or LXMLWebScrapingStrategy()
|
||||||
# Normalize proxy_config: single ProxyConfig stored as-is, list stored as list
|
self.proxy_config = proxy_config # runs through property setter
|
||||||
if isinstance(proxy_config, list):
|
|
||||||
normalized = []
|
|
||||||
for p in proxy_config:
|
|
||||||
if p is None or p == "direct":
|
|
||||||
normalized.append(None)
|
|
||||||
elif isinstance(p, dict):
|
|
||||||
normalized.append(ProxyConfig.from_dict(p))
|
|
||||||
elif isinstance(p, str):
|
|
||||||
normalized.append(ProxyConfig.from_string(p))
|
|
||||||
else:
|
|
||||||
normalized.append(p)
|
|
||||||
self.proxy_config = normalized
|
|
||||||
elif isinstance(proxy_config, dict):
|
|
||||||
self.proxy_config = ProxyConfig.from_dict(proxy_config)
|
|
||||||
elif isinstance(proxy_config, str):
|
|
||||||
self.proxy_config = ProxyConfig.from_string(proxy_config)
|
|
||||||
else:
|
|
||||||
self.proxy_config = proxy_config # ProxyConfig or None
|
|
||||||
|
|
||||||
self.proxy_rotation_strategy = proxy_rotation_strategy
|
self.proxy_rotation_strategy = proxy_rotation_strategy
|
||||||
|
|
||||||
@@ -1681,6 +1663,37 @@ class CrawlerRunConfig():
|
|||||||
self._compile_c4a_script()
|
self._compile_c4a_script()
|
||||||
|
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _normalize_proxy_config(value):
|
||||||
|
"""Normalize proxy_config to ProxyConfig, list of ProxyConfig/None, or None."""
|
||||||
|
if isinstance(value, list):
|
||||||
|
normalized = []
|
||||||
|
for p in value:
|
||||||
|
if p is None or p == "direct":
|
||||||
|
normalized.append(None)
|
||||||
|
elif isinstance(p, dict):
|
||||||
|
normalized.append(ProxyConfig.from_dict(p))
|
||||||
|
elif isinstance(p, str):
|
||||||
|
normalized.append(ProxyConfig.from_string(p))
|
||||||
|
else:
|
||||||
|
normalized.append(p)
|
||||||
|
return normalized
|
||||||
|
elif isinstance(value, dict):
|
||||||
|
return ProxyConfig.from_dict(value)
|
||||||
|
elif isinstance(value, str):
|
||||||
|
if value == "direct":
|
||||||
|
return None
|
||||||
|
return ProxyConfig.from_string(value)
|
||||||
|
return value # ProxyConfig or None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def proxy_config(self):
|
||||||
|
return self._proxy_config
|
||||||
|
|
||||||
|
@proxy_config.setter
|
||||||
|
def proxy_config(self, value):
|
||||||
|
self._proxy_config = CrawlerRunConfig._normalize_proxy_config(value)
|
||||||
|
|
||||||
def _get_proxy_list(self) -> list:
|
def _get_proxy_list(self) -> list:
|
||||||
"""Normalize proxy_config to a list for the retry loop."""
|
"""Normalize proxy_config to a list for the retry loop."""
|
||||||
if self.proxy_config is None:
|
if self.proxy_config is None:
|
||||||
|
|||||||
Reference in New Issue
Block a user