ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved database management and error handling, updated README instructions, refined .gitignore, enhanced async web crawling capabilities, and updated dependencies.

Browse Source
This commit is contained in:
unclecode
2024-11-04 13:22:13 +08:00
parent 62a86dbe8d
commit 54d5a3a259
11 changed files with 461 additions and 669 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
crawl4ai/async_webcrawler.py
View File

@@ -48,7 +48,8 @@ class AsyncWebCrawler:
async def awarmup(self):
if self.verbose:
print("[LOG] 🌤️ Warming up the AsyncWebCrawler")
await async_db_manager.ainit_db()
# await async_db_manager.ainit_db()
await async_db_manager.initialize()
await self.arun(
url="https://google.com/",
word_count_threshold=5,
@@ -125,6 +126,7 @@ class AsyncWebCrawler:
verbose,
bool(cached),
async_response=async_response,
bypass_cache=bypass_cache,
**kwargs,
)
crawl_result.status_code = async_response.status_code if async_response else 200
@@ -243,7 +245,7 @@ class AsyncWebCrawler:
screenshot = None if not screenshot else screenshot
if not is_cached:
if not is_cached or kwargs.get("bypass_cache", False) or self.always_by_pass_cache:
await async_db_manager.acache_url(
url,
html,
@@ -274,7 +276,8 @@ class AsyncWebCrawler:
)
async def aclear_cache(self):
await async_db_manager.aclear_db()
# await async_db_manager.aclear_db()
await async_db_manager.cleanup()
async def aflush_cache(self):
await async_db_manager.aflush_db()