feat(config): add streaming support and config cloning

Add streaming capability to crawler configurations and introduce clone() methods
for both BrowserConfig and CrawlerRunConfig to support immutable config updates.
Move stream parameter from arun_many() method to CrawlerRunConfig.

BREAKING CHANGE: Removed stream parameter from AsyncWebCrawler.arun_many() method.
Use config.stream=True instead.
This commit is contained in:
UncleCode
2025-01-19 17:51:47 +08:00
parent 1221be30a3
commit 91463e34f1
3 changed files with 87 additions and 10 deletions

View File

@@ -33,9 +33,7 @@ async def test_crawler():
print("\n=== Testing Streaming Mode ===")
async for result in await crawler.arun_many(
urls=urls,
config=crawler_config,
stream=True,
verbose=True
config=crawler_config.clone(stream=True),
):
print(f"Received result for: {result.url} - Success: {result.success}")
@@ -43,8 +41,6 @@ async def test_crawler():
results = await crawler.arun_many(
urls=urls,
config=crawler_config,
stream=False,
verbose=True
)
print(f"Received all {len(results)} results at once")
for result in results: