updated definition of can_process_url to include dept as an argument, as it's needed to skip filters for start_url

This commit is contained in:
Aravind Karnam
2024-11-26 18:26:57 +05:30
parent ff731e4ea1
commit 2f5e0598bb

View File

@@ -32,7 +32,7 @@ class ScraperStrategy(ABC):
pass
@abstractmethod
async def can_process_url(self, url: str) -> bool:
async def can_process_url(self, url: str, depth: int) -> bool:
"""Check if URL can be processed based on strategy rules"""
pass