feat(profiles): add CLI command for crawling with browser profiles

Adds new functionality to crawl websites using saved browser profiles directly from the CLI.
This includes:
- New CLI option to use profiles for crawling
- Helper functions for profile-based crawling
- Fixed type hints for config parameters
- Updated example to show browser window by default

This makes it easier for users to leverage saved browser profiles for crawling without writing code.
This commit is contained in:
UncleCode
2025-03-02 21:33:33 +08:00
parent 95175cb394
commit c612f9a852
4 changed files with 87 additions and 7 deletions

View File

@@ -16,7 +16,7 @@ class DeepCrawlDecorator:
def __call__(self, original_arun):
@wraps(original_arun)
async def wrapped_arun(url: str, config: Optional[CrawlerRunConfig] = None, **kwargs):
async def wrapped_arun(url: str, config: CrawlerRunConfig = None, **kwargs):
# If deep crawling is already active, call the original method to avoid recursion.
if config and config.deep_crawl_strategy and not self.deep_crawl_active.get():
token = self.deep_crawl_active.set(True)