updated definition of can_process_url to include dept as an argument, as it's needed to skip filters for start_url
This commit is contained in:
@@ -32,7 +32,7 @@ class ScraperStrategy(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def can_process_url(self, url: str) -> bool:
|
||||
async def can_process_url(self, url: str, depth: int) -> bool:
|
||||
"""Check if URL can be processed based on strategy rules"""
|
||||
pass
|
||||
|
||||
|
||||
Reference in New Issue
Block a user