feat(profiles): add CLI command for crawling with browser profiles

Adds new functionality to crawl websites using saved browser profiles directly from the CLI.
This includes:
- New CLI option to use profiles for crawling
- Helper functions for profile-based crawling
- Fixed type hints for config parameters
- Updated example to show browser window by default

This makes it easier for users to leverage saved browser profiles for crawling without writing code.
This commit is contained in:
UncleCode
2025-03-02 21:33:33 +08:00
parent 95175cb394
commit c612f9a852
4 changed files with 87 additions and 7 deletions

View File

@@ -222,7 +222,7 @@ class AsyncWebCrawler:
async def arun(
self,
url: str,
config: Optional[CrawlerRunConfig] = None,
config: CrawlerRunConfig = None,
# Legacy parameters maintained for backwards compatibility
word_count_threshold=MIN_WORD_THRESHOLD,
extraction_strategy: ExtractionStrategy = None,
@@ -270,7 +270,7 @@ class AsyncWebCrawler:
Returns:
CrawlResult: The result of crawling and processing
"""
crawler_config = config
crawler_config = config or CrawlerRunConfig()
if not isinstance(url, str) or not url:
raise ValueError("Invalid URL, make sure the URL is a non-empty string")