Refactor content scrapping strategy and improve error handling

2024-10-20 19:11:18 +08:00
parent 6ec4cb33ca
commit 1dd36f9035
2 changed files with 61 additions and 53 deletions
--- a/crawl4ai/async_webcrawler.py
+++ b/crawl4ai/async_webcrawler.py
@@ -187,7 +187,7 @@ class AsyncWebCrawler:
            t1 = time.time()
            scrapping_strategy = WebScrappingStrategy()
            # result = await scrapping_strategy.ascrap(
-            result = await scrapping_strategy.scrap(
+            result = scrapping_strategy.scrap(
                url,
                html,
                word_count_threshold=word_count_threshold,