a9e24307cc
* fix: Update export of URLPatternFilter * chore: Add dependancy for cchardet in requirements * docs: Update example for deep crawl in release note for v0.5 * Docs: update the example for memory dispatcher * docs: updated example for crawl strategies * Refactor: Removed wrapping in if __name__==main block since this is a markdown file. * chore: removed cchardet from dependancy list, since unclecode is planning to remove it * docs: updated the example for proxy rotation to a working example * feat: Introduced ProxyConfig param * Add tutorial for deep crawl & update contributor list for bug fixes in feb alpha-1 * chore: update and test new dependancies * feat:Make PyPDF2 a conditional dependancy * updated tutorial and release note for v0.5 * docs: update docs for deep crawl, and fix a typo in docker-deployment markdown filename * refactor: 1. Deprecate markdown_v2 2. Make markdown backward compatible to behave as a string when needed. 3. Fix LlmConfig usage in cli 4. Deprecate markdown_v2 in cli 5. Update AsyncWebCrawler for changes in CrawlResult * fix: Bug in serialisation of markdown in acache_url * Refactor: Added deprecation errors for fit_html and fit_markdown directly on markdown. Now access them via markdown * fix: remove deprecated markdown_v2 from docker * Refactor: remove deprecated fit_markdown and fit_html from result * refactor: fix cache retrieval for markdown as a string * chore: update all docs, examples and tests with deprecation announcements for markdown_v2, fit_html, fit_markdown
114 lines
3.7 KiB
Python
114 lines
3.7 KiB
Python
import os
|
|
from typing import Dict, List, Optional
|
|
|
|
|
|
class ProxyConfig:
|
|
def __init__(
|
|
self,
|
|
server: str,
|
|
username: Optional[str] = None,
|
|
password: Optional[str] = None,
|
|
ip: Optional[str] = None,
|
|
):
|
|
"""Configuration class for a single proxy.
|
|
|
|
Args:
|
|
server: Proxy server URL (e.g., "http://127.0.0.1:8080")
|
|
username: Optional username for proxy authentication
|
|
password: Optional password for proxy authentication
|
|
ip: Optional IP address for verification purposes
|
|
"""
|
|
self.server = server
|
|
self.username = username
|
|
self.password = password
|
|
|
|
# Extract IP from server if not explicitly provided
|
|
self.ip = ip or self._extract_ip_from_server()
|
|
|
|
def _extract_ip_from_server(self) -> Optional[str]:
|
|
"""Extract IP address from server URL."""
|
|
try:
|
|
# Simple extraction assuming http://ip:port format
|
|
if "://" in self.server:
|
|
parts = self.server.split("://")[1].split(":")
|
|
return parts[0]
|
|
else:
|
|
parts = self.server.split(":")
|
|
return parts[0]
|
|
except Exception:
|
|
return None
|
|
|
|
@staticmethod
|
|
def from_string(proxy_str: str) -> "ProxyConfig":
|
|
"""Create a ProxyConfig from a string in the format 'ip:port:username:password'."""
|
|
parts = proxy_str.split(":")
|
|
if len(parts) == 4: # ip:port:username:password
|
|
ip, port, username, password = parts
|
|
return ProxyConfig(
|
|
server=f"http://{ip}:{port}",
|
|
username=username,
|
|
password=password,
|
|
ip=ip
|
|
)
|
|
elif len(parts) == 2: # ip:port only
|
|
ip, port = parts
|
|
return ProxyConfig(
|
|
server=f"http://{ip}:{port}",
|
|
ip=ip
|
|
)
|
|
else:
|
|
raise ValueError(f"Invalid proxy string format: {proxy_str}")
|
|
|
|
@staticmethod
|
|
def from_dict(proxy_dict: Dict) -> "ProxyConfig":
|
|
"""Create a ProxyConfig from a dictionary."""
|
|
return ProxyConfig(
|
|
server=proxy_dict.get("server"),
|
|
username=proxy_dict.get("username"),
|
|
password=proxy_dict.get("password"),
|
|
ip=proxy_dict.get("ip")
|
|
)
|
|
|
|
@staticmethod
|
|
def from_env(env_var: str = "PROXIES") -> List["ProxyConfig"]:
|
|
"""Load proxies from environment variable.
|
|
|
|
Args:
|
|
env_var: Name of environment variable containing comma-separated proxy strings
|
|
|
|
Returns:
|
|
List of ProxyConfig objects
|
|
"""
|
|
proxies = []
|
|
try:
|
|
proxy_list = os.getenv(env_var, "").split(",")
|
|
for proxy in proxy_list:
|
|
if not proxy:
|
|
continue
|
|
proxies.append(ProxyConfig.from_string(proxy))
|
|
except Exception as e:
|
|
print(f"Error loading proxies from environment: {e}")
|
|
return proxies
|
|
|
|
def to_dict(self) -> Dict:
|
|
"""Convert to dictionary representation."""
|
|
return {
|
|
"server": self.server,
|
|
"username": self.username,
|
|
"password": self.password,
|
|
"ip": self.ip
|
|
}
|
|
|
|
def clone(self, **kwargs) -> "ProxyConfig":
|
|
"""Create a copy of this configuration with updated values.
|
|
|
|
Args:
|
|
**kwargs: Key-value pairs of configuration options to update
|
|
|
|
Returns:
|
|
ProxyConfig: A new instance with the specified updates
|
|
"""
|
|
config_dict = self.to_dict()
|
|
config_dict.update(kwargs)
|
|
return ProxyConfig.from_dict(config_dict)
|