From 9109ecd8fc50ce9c9b87bd8e58aa863648556f82 Mon Sep 17 00:00:00 2001 From: Aravind Karnam Date: Tue, 18 Mar 2025 15:26:20 +0530 Subject: [PATCH] chore: Raise an exception with clear messaging when body tag is missing in the fetched html. The message should warn users to add appropriate wait_for condition to wait until body tag is loaded into DOM. fixes: https://github.com/unclecode/crawl4ai/issues/804 --- crawl4ai/content_scraping_strategy.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crawl4ai/content_scraping_strategy.py b/crawl4ai/content_scraping_strategy.py index ef622abe..215e7cda 100644 --- a/crawl4ai/content_scraping_strategy.py +++ b/crawl4ai/content_scraping_strategy.py @@ -862,6 +862,8 @@ class WebScrapingStrategy(ContentScrapingStrategy): parser_type = kwargs.get("parser", "lxml") soup = BeautifulSoup(html, parser_type) body = soup.body + if body is None: + raise Exception("'' tag is not found in fetched html. Consider adding wait_for=\"css:body\" to wait for body tag to be loaded into DOM.") base_domain = get_base_domain(url) try: