feat(cache): introduce CacheMode and CacheContext for enhanced caching behavior

chore(requirements): add colorama dependency
refactor(config): add SHOW_DEPRECATION_WARNINGS flag and clean up code
fix(docs): update example scripts for clarity and consistency
This commit is contained in:
UncleCode
2024-11-17 15:30:56 +08:00
parent 4b45b28f25
commit 3a66aa8a60
10 changed files with 979 additions and 95 deletions

View File

@@ -48,8 +48,8 @@ class Crawl4AiTester:
def test_docker_deployment(version="basic"):
tester = Crawl4AiTester(
base_url="http://localhost:11235" ,
# base_url="https://crawl4ai-sby74.ondigitalocean.app",
api_token="test"
# base_url="https://api.crawl4ai.com" # just for example
# api_token="test" # just for example
)
print(f"Testing Crawl4AI Docker {version} version")
@@ -69,6 +69,7 @@ def test_docker_deployment(version="basic"):
# Test cases based on version
test_basic_crawl(tester)
test_basic_crawl(tester)
test_basic_crawl_sync(tester)
# if version in ["full", "transformer"]:

View File

@@ -71,12 +71,12 @@ async def use_proxy():
"Note: Replace 'http://your-proxy-url:port' with a working proxy to run this example."
)
# Uncomment and modify the following lines to use a proxy
# async with AsyncWebCrawler(verbose=True, proxy="http://your-proxy-url:port") as crawler:
# result = await crawler.arun(
# url="https://www.nbcnews.com/business",
# bypass_cache=True
# )
# print(result.markdown[:500]) # Print first 500 characters
async with AsyncWebCrawler(verbose=True, proxy="http://your-proxy-url:port") as crawler:
result = await crawler.arun(
url="https://www.nbcnews.com/business",
bypass_cache=True
)
print(result.markdown[:500]) # Print first 500 characters
async def capture_and_save_screenshot(url: str, output_path: str):
async with AsyncWebCrawler(verbose=True) as crawler: