updated definition of can_process_url to include dept as an argument, as it's needed to skip filters for start_url
This commit is contained in:
@@ -32,7 +32,7 @@ class ScraperStrategy(ABC):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def can_process_url(self, url: str) -> bool:
|
async def can_process_url(self, url: str, depth: int) -> bool:
|
||||||
"""Check if URL can be processed based on strategy rules"""
|
"""Check if URL can be processed based on strategy rules"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user