From 6e3c0483286b2902ab814630ab199449249176f6 Mon Sep 17 00:00:00 2001 From: UncleCode Date: Thu, 13 Mar 2025 22:30:38 +0800 Subject: [PATCH] feat(api): refactor crawl request handling to streamline single and multiple URL processing --- deploy/docker/api.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 4c7e17d2..305e8a31 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -2,6 +2,7 @@ import os import json import asyncio from typing import List, Tuple +from functools import partial import logging from typing import Optional, AsyncGenerator @@ -389,19 +390,9 @@ async def handle_crawl_request( async with AsyncWebCrawler(config=browser_config) as crawler: results = [] - if len(urls) == 1: - results = await crawler.arun( - url=urls[0], - config=crawler_config, - dispatcher=dispatcher - ) - else: - results = await crawler.arun_many( - urls=urls, - config=crawler_config, - dispatcher=dispatcher - ) - + func = getattr(crawler, "arun" if len(urls) == 1 else "arun_many") + partial_func = partial(func, urls[0] if len(urls) == 1 else urls, config=crawler_config, dispatcher=dispatcher) + results = await partial_func() return { "success": True, "results": [result.model_dump() for result in results]