ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'next' into 2025-MAR-ALPHA-1

Browse Source
This commit is contained in:
Aravind Karnam
2025-04-17 10:50:02 +05:30
parent dcc265458c 94d486579c
commit eed7f88f29
38 changed files with 5574 additions and 878 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
deploy/docker/api.py
View File

@@ -391,21 +391,25 @@ async def handle_crawl_request(
)
)
async with AsyncWebCrawler(config=browser_config) as crawler:
results = []
func = getattr(crawler, "arun" if len(urls) == 1 else "arun_many")
partial_func = partial(func,
urls[0] if len(urls) == 1 else urls,
config=crawler_config,
dispatcher=dispatcher)
results = await partial_func()
return {
"success": True,
"results": [result.model_dump() for result in results]
}
crawler: AsyncWebCrawler = AsyncWebCrawler(config=browser_config)
await crawler.start()
results = []
func = getattr(crawler, "arun" if len(urls) == 1 else "arun_many")
partial_func = partial(func,
urls[0] if len(urls) == 1 else urls,
config=crawler_config,
dispatcher=dispatcher)
results = await partial_func()
await crawler.close()
return {
"success": True,
"results": [result.model_dump() for result in results]
}
except Exception as e:
logger.error(f"Crawl error: {str(e)}", exc_info=True)
if 'crawler' in locals():
await crawler.close()
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=str(e)