ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/unclecode/crawl4ai

Browse Source
This commit is contained in:
UncleCode
2024-10-27 19:28:05 +08:00
parent dbb587d681 32f57c49d6
commit ff9149b5c9
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
crawl4ai/async_webcrawler.py
View File

@@ -23,13 +23,14 @@ class AsyncWebCrawler:
self, self,
crawler_strategy: Optional[AsyncCrawlerStrategy] = None, crawler_strategy: Optional[AsyncCrawlerStrategy] = None,
always_by_pass_cache: bool = False, always_by_pass_cache: bool = False,
base_directory: str = str(Path.home()),
**kwargs, **kwargs,
): ):
self.crawler_strategy = crawler_strategy or AsyncPlaywrightCrawlerStrategy( self.crawler_strategy = crawler_strategy or AsyncPlaywrightCrawlerStrategy(
**kwargs **kwargs
) )
self.always_by_pass_cache = always_by_pass_cache self.always_by_pass_cache = always_by_pass_cache
self.crawl4ai_folder = os.path.join(Path.home(), ".crawl4ai") self.crawl4ai_folder = os.path.join(base_directory, ".crawl4ai")
os.makedirs(self.crawl4ai_folder, exist_ok=True) os.makedirs(self.crawl4ai_folder, exist_ok=True)
os.makedirs(f"{self.crawl4ai_folder}/cache", exist_ok=True) os.makedirs(f"{self.crawl4ai_folder}/cache", exist_ok=True)
self.ready = False self.ready = False