diff --git a/crawl4ai/content_scraping_strategy.py b/crawl4ai/content_scraping_strategy.py index 3510f64f..b5ece9ba 100644 --- a/crawl4ai/content_scraping_strategy.py +++ b/crawl4ai/content_scraping_strategy.py @@ -696,6 +696,13 @@ class LXMLWebScrapingStrategy(ContentScrapingStrategy): success = True try: + # Extract metadata FIRST from the original HTML to avoid issues with modified content. + try: + meta = extract_metadata_using_lxml(html, None) # Pass the original HTML + except Exception as e: + self._log("error", f"Error extracting metadata: {str(e)}", "SCRAPE") + meta = {} + doc = lhtml.document_fromstring(html) # Match BeautifulSoup's behavior of using body or full doc # body = doc.xpath('//body')[0] if doc.xpath('//body') else doc @@ -736,14 +743,14 @@ class LXMLWebScrapingStrategy(ContentScrapingStrategy): "error", f"Error with excluded CSS selector: {str(e)}", "SCRAPE" ) - # Extract metadata before any content filtering - try: - meta = extract_metadata_using_lxml( - "", doc - ) # Using same function as BeautifulSoup version - except Exception as e: - self._log("error", f"Error extracting metadata: {str(e)}", "SCRAPE") - meta = {} + # # Extract metadata before any content filtering + # try: + # meta = extract_metadata_using_lxml( + # "", doc + # ) # Using same function as BeautifulSoup version + # except Exception as e: + # self._log("error", f"Error extracting metadata: {str(e)}", "SCRAPE") + # meta = {} content_element = None if target_elements: diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py index d8b366d9..ef5df62f 100644 --- a/crawl4ai/utils.py +++ b/crawl4ai/utils.py @@ -1487,8 +1487,29 @@ def extract_metadata_using_lxml(html, doc=None): head = head[0] # Title - using XPath + # title = head.xpath(".//title/text()") + # metadata["title"] = title[0].strip() if title else None + + # === Title Extraction - New Approach === + # Attempt to extract