diff --git a/deploy/docker/server.py b/deploy/docker/server.py index b79324da..57d78d06 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -482,9 +482,14 @@ async def crawl( ): """ Crawl a list of URLs and return the results as JSON. + For streaming responses, use /crawl/stream endpoint. """ if not crawl_request.urls: raise HTTPException(400, "At least one URL required") + # Check whether it is a redirection for a streaming request + crawler_config = CrawlerRunConfig.load(crawl_request.crawler_config) + if crawler_config.stream: + return await stream_process(crawl_request=crawl_request) results = await handle_crawl_request( urls=crawl_request.urls, browser_config=crawl_request.browser_config, @@ -506,12 +511,16 @@ async def crawl_stream( ): if not crawl_request.urls: raise HTTPException(400, "At least one URL required") + + return await stream_process(crawl_request=crawl_request) + +async def stream_process(crawl_request: CrawlRequest): crawler, gen = await handle_stream_crawl_request( urls=crawl_request.urls, browser_config=crawl_request.browser_config, crawler_config=crawl_request.crawler_config, config=config, - ) +) return StreamingResponse( stream_results(crawler, gen), media_type="application/x-ndjson", diff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.html index e01fc857..553e6765 100644 --- a/deploy/docker/static/playground/index.html +++ b/deploy/docker/static/playground/index.html @@ -371,7 +371,7 @@
- +