feat(config): add streaming support and config cloning
Add streaming capability to crawler configurations and introduce clone() methods for both BrowserConfig and CrawlerRunConfig to support immutable config updates. Move stream parameter from arun_many() method to CrawlerRunConfig. BREAKING CHANGE: Removed stream parameter from AsyncWebCrawler.arun_many() method. Use config.stream=True instead.
This commit is contained in:
@@ -33,9 +33,7 @@ async def test_crawler():
|
||||
print("\n=== Testing Streaming Mode ===")
|
||||
async for result in await crawler.arun_many(
|
||||
urls=urls,
|
||||
config=crawler_config,
|
||||
stream=True,
|
||||
verbose=True
|
||||
config=crawler_config.clone(stream=True),
|
||||
):
|
||||
print(f"Received result for: {result.url} - Success: {result.success}")
|
||||
|
||||
@@ -43,8 +41,6 @@ async def test_crawler():
|
||||
results = await crawler.arun_many(
|
||||
urls=urls,
|
||||
config=crawler_config,
|
||||
stream=False,
|
||||
verbose=True
|
||||
)
|
||||
print(f"Received all {len(results)} results at once")
|
||||
for result in results:
|
||||
|
||||
Reference in New Issue
Block a user