Fix: enhance fallback scoring for failed head extraction in LinkPreview. ref #1638

Fix: Wrong URL variable used for extraction of raw html

Dockerfile

@@ -167,6 +167,11 @@ RUN mkdir -p /home/appuser/.cache/ms-playwright \
 
 RUN crawl4ai-doctor
 
+# Ensure all cache directories belong to appuser
+# This fixes permission issues with .cache/url_seeder and other runtime cache dirs
+RUN mkdir -p /home/appuser/.cache \
+    && chown -R appuser:appuser /home/appuser/.cache
+
 # Copy application code
 COPY deploy/docker/* ${APP_HOME}/
 

crawl4ai/adaptive_crawler.py

@@ -728,18 +728,18 @@ class EmbeddingStrategy(CrawlStrategy):
         provider = llm_config_dict.get('provider', 'openai/gpt-4o-mini') if llm_config_dict else 'openai/gpt-4o-mini'
         api_token = llm_config_dict.get('api_token') if llm_config_dict else None
         
-        # response = perform_completion_with_backoff(
-        #     provider=provider,
-        #     prompt_with_variables=prompt,
-        #     api_token=api_token,
-        #     json_response=True
-        # )
+        response = perform_completion_with_backoff(
+            provider=provider,
+            prompt_with_variables=prompt,
+            api_token=api_token,
+            json_response=True
+        )
         
-        # variations = json.loads(response.choices[0].message.content)
+        variations = json.loads(response.choices[0].message.content)
         
         
         # # Mock data with more variations for split
-        variations ={'queries': ['what are the best vegetables to use in fried rice?', 'how do I make vegetable fried rice from scratch?', 'can you provide a quick recipe for vegetable fried rice?', 'what cooking techniques are essential for perfect fried rice with vegetables?', 'how to add flavor to vegetable fried rice?', 'are there any tips for making healthy fried rice with vegetables?']}
+        # variations ={'queries': ['what are the best vegetables to use in fried rice?', 'how do I make vegetable fried rice from scratch?', 'can you provide a quick recipe for vegetable fried rice?', 'what cooking techniques are essential for perfect fried rice with vegetables?', 'how to add flavor to vegetable fried rice?', 'are there any tips for making healthy fried rice with vegetables?']}
         
         
         # variations = {'queries': [

crawl4ai/async_webcrawler.py

@@ -617,17 +617,17 @@ class AsyncWebCrawler:
                 else config.chunking_strategy
             )
             sections = chunking.chunk(content)
-            # extracted_content = config.extraction_strategy.run(url, sections)
+            # extracted_content = config.extraction_strategy.run(_url, sections)
 
             # Use async version if available for better parallelism
             if hasattr(config.extraction_strategy, 'arun'):
-                extracted_content = await config.extraction_strategy.arun(url, sections)
+                extracted_content = await config.extraction_strategy.arun(_url, sections)
             else:
                 # Fallback to sync version run in thread pool to avoid blocking
                 extracted_content = await asyncio.to_thread(
                     config.extraction_strategy.run, url, sections
                 )
-
+                
             extracted_content = json.dumps(
                 extracted_content, indent=4, default=str, ensure_ascii=False
             )

crawl4ai/browser_manager.py

@@ -674,11 +674,6 @@ class BrowserManager:
                 self.default_context = await self.create_browser_context()
             await self.setup_context(self.default_context)
         else:
-            # Handle --disable-web-security requiring a separate user data directory
-            if "--disable-web-security" in (self.config.extra_args or []) and not self.config.user_data_dir:
-                import tempfile
-                self.config.user_data_dir = tempfile.mkdtemp()
-            
             browser_args = self._build_browser_args()
 
             # Launch appropriate browser type
@@ -687,15 +682,9 @@ class BrowserManager:
             elif self.config.browser_type == "webkit":
                 self.browser = await self.playwright.webkit.launch(**browser_args)
             else:
-                if "--disable-web-security" in (self.config.extra_args or []):
-                    # Use persistent context for --disable-web-security
-                    browser_args["args"] = [arg for arg in browser_args["args"] if not arg.startswith("--user-data-dir")]
-                    self.default_context = await self.playwright.chromium.launch_persistent_context(self.config.user_data_dir, **browser_args)
-                    self.browser = self.default_context
-                    self.config.use_managed_browser = True  # Treat as managed for get_page logic
-                else:
-                    self.browser = await self.playwright.chromium.launch(**browser_args)
-                    self.default_context = self.browser
+                self.browser = await self.playwright.chromium.launch(**browser_args)
+
+            self.default_context = self.browser
 
     async def _verify_cdp_ready(self, cdp_url: str) -> bool:
         """Verify CDP endpoint is ready with exponential backoff"""
@@ -759,9 +748,6 @@ class BrowserManager:
         if self.config.extra_args:
             args.extend(self.config.extra_args)
 
-        if self.config.user_data_dir:
-            args.append(f"--user-data-dir={self.config.user_data_dir}")
-
         # Deduplicate args
         args = list(dict.fromkeys(args))
         

crawl4ai/link_preview.py

@@ -336,8 +336,40 @@ class LinkPreview:
                 
                 updated_internal.append(updated_link)
             else:
-                # Keep original link unchanged
-                updated_internal.append(link)
+                # # Keep original link unchanged
+                # updated_internal.append(link)
+                
+                # Head extraction failed - calculate fallback scores
+                # Use URL-based scoring if query provided
+                contextual_score = None
+                if config.link_preview_config and config.link_preview_config.query:
+                    # Calculate URL-based relevance score as fallback
+                    contextual_score = self.seeder._calculate_url_relevance_score(
+                        config.link_preview_config.query,
+                        link.href
+                    )
+
+                # Create updated link with fallback scoring
+                updated_link = Link(
+                    href=link.href,
+                    text=link.text,
+                    title=link.title,
+                    base_domain=link.base_domain,
+                    head_data=None,  # No head data available
+                    head_extraction_status="failed",
+                    intrinsic_score=getattr(link, 'intrinsic_score', None),
+                    contextual_score=contextual_score
+                )
+
+                # Calculate total score even without head data
+                updated_link.total_score = calculate_total_score(
+                    intrinsic_score=updated_link.intrinsic_score,
+                    contextual_score=updated_link.contextual_score,
+                    score_links_enabled=getattr(config, 'score_links', False),
+                    query_provided=bool(config.link_preview_config and config.link_preview_config.query)
+                )
+
+                updated_internal.append(updated_link)
         
         # Update external links
         updated_external = []
@@ -374,8 +406,40 @@ class LinkPreview:
                 
                 updated_external.append(updated_link)
             else:
-                # Keep original link unchanged
-                updated_external.append(link)
+                # # Keep original link unchanged
+                # updated_external.append(link)
+                
+                # Head extraction failed - calculate fallback scores
+                # Use URL-based scoring if query provided
+                contextual_score = None
+                if config.link_preview_config and config.link_preview_config.query:
+                    # Calculate URL-based relevance score as fallback
+                    contextual_score = self.seeder._calculate_url_relevance_score(
+                        config.link_preview_config.query,
+                        link.href
+                    )
+
+                # Create updated link with fallback scoring
+                updated_link = Link(
+                    href=link.href,
+                    text=link.text,
+                    title=link.title,
+                    base_domain=link.base_domain,
+                    head_data=None,  # No head data available
+                    head_extraction_status="failed",
+                    intrinsic_score=getattr(link, 'intrinsic_score', None),
+                    contextual_score=contextual_score
+                )
+
+                # Calculate total score even without head data
+                updated_link.total_score = calculate_total_score(
+                    intrinsic_score=updated_link.intrinsic_score,
+                    contextual_score=updated_link.contextual_score,
+                    score_links_enabled=getattr(config, 'score_links', False),
+                    query_provided=bool(config.link_preview_config and config.link_preview_config.query)
+                )
+
+                updated_external.append(updated_link)
         
         # Sort links by relevance score if available
         if any(hasattr(link, 'head_data') and link.head_data and 'relevance_score' in link.head_data 

tests/general/test_async_webcrawler.py

@@ -9,6 +9,21 @@ from crawl4ai import (
     RateLimiter,
     CacheMode
 )
+from crawl4ai.extraction_strategy import ExtractionStrategy
+
+class MockExtractionStrategy(ExtractionStrategy):
+    """Mock extraction strategy for testing URL parameter handling"""
+
+    def __init__(self):
+        super().__init__()
+        self.run_calls = []
+
+    def extract(self, url: str, html: str, *args, **kwargs):
+        return [{"test": "data"}]
+
+    def run(self, url: str, sections: List[str], *args, **kwargs):
+        self.run_calls.append(url)
+        return super().run(url, sections, *args, **kwargs)
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("viewport", [
@@ -142,8 +157,72 @@ async def test_error_handling(error_url):
         assert not result.success
         assert result.error_message is not None
 
+@pytest.mark.asyncio
+async def test_extraction_strategy_run_with_regular_url():
+    """
+    Regression test for extraction_strategy.run URL parameter handling with regular URLs.
+
+    This test verifies that when is_raw_html=False (regular URL),
+    extraction_strategy.run is called with the actual URL.
+    """
+    browser_config = BrowserConfig(
+        browser_type="chromium",
+        headless=True
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        mock_strategy = MockExtractionStrategy()
+
+        # Test regular URL (is_raw_html=False)
+        regular_url = "https://example.com"
+        result = await crawler.arun(
+            url=regular_url,
+            config=CrawlerRunConfig(
+                page_timeout=30000,
+                extraction_strategy=mock_strategy,
+                cache_mode=CacheMode.BYPASS
+            )
+        )
+
+        assert result.success
+        assert len(mock_strategy.run_calls) == 1
+        assert mock_strategy.run_calls[0] == regular_url, f"Expected '{regular_url}', got '{mock_strategy.run_calls[0]}'"
+
+@pytest.mark.asyncio
+async def test_extraction_strategy_run_with_raw_html():
+    """
+    Regression test for extraction_strategy.run URL parameter handling with raw HTML.
+
+    This test verifies that when is_raw_html=True (URL starts with "raw:"),
+    extraction_strategy.run is called with "Raw HTML" instead of the actual URL.
+    """
+    browser_config = BrowserConfig(
+        browser_type="chromium",
+        headless=True
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        mock_strategy = MockExtractionStrategy()
+
+        # Test raw HTML URL (is_raw_html=True automatically set)
+        raw_html_url = "raw:<html><body><h1>Test HTML</h1><p>This is a test.</p></body></html>"
+        result = await crawler.arun(
+            url=raw_html_url,
+            config=CrawlerRunConfig(
+                page_timeout=30000,
+                extraction_strategy=mock_strategy,
+                cache_mode=CacheMode.BYPASS
+            )
+        )
+
+        assert result.success
+        assert len(mock_strategy.run_calls) == 1
+        assert mock_strategy.run_calls[0] == "Raw HTML", f"Expected 'Raw HTML', got '{mock_strategy.run_calls[0]}'"
+
 if __name__ == "__main__":
     asyncio.run(test_viewport_config((1024, 768)))
     asyncio.run(test_memory_management())
     asyncio.run(test_rate_limiting())
-    asyncio.run(test_javascript_execution())
+    asyncio.run(test_javascript_execution())
+    asyncio.run(test_extraction_strategy_run_with_regular_url())
+    asyncio.run(test_extraction_strategy_run_with_raw_html())

tests/test_browser_manager_cors.py

@@ -1,73 +0,0 @@
-import os
-import sys
-import pytest
-
-# Add the parent directory to the Python path
-parent_dir = os.path.dirname(
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-)
-sys.path.append(parent_dir)
-
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
-
-
-@pytest.mark.asyncio
-async def test_normal_browser_launch():
-    """Test that the browser manager launches normally without --disable-web-security"""
-    async with AsyncWebCrawler() as crawler:
-        result = await crawler.arun(url="https://example.com", bypass_cache=True)
-        assert result.success
-        assert result.html
-        assert result.markdown
-
-
-@pytest.mark.asyncio
-async def test_cors_bypass_with_disable_web_security():
-    """Test that --disable-web-security allows XMLHttpRequest to bypass CORS"""
-    browser_config = BrowserConfig(
-        extra_args=['--disable-web-security'],
-        headless=True  # Run headless for test
-    )
-
-    # JS code that attempts XMLHttpRequest to a cross-origin URL that normally blocks CORS
-    js_code = """
-    var xhr = new XMLHttpRequest();
-    xhr.open('GET', 'https://raw.githubusercontent.com/tatsu-lab/alpaca_eval/main/docs/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv', false);
-    xhr.send();
-    if (xhr.status == 200) {
-        return {success: true, length: xhr.responseText.length};
-    } else {
-        return {success: false, status: xhr.status, error: xhr.statusText};
-    }
-    """
-
-    crawler_config = CrawlerRunConfig(js_code=js_code)
-
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        result = await crawler.arun(url="https://example.com", config=crawler_config, bypass_cache=True)
-        assert result.success, f"Crawl failed: {result.error_message}"
-        js_result = result.js_execution_result
-        assert js_result is not None, "JS execution result is None"
-        assert js_result.get('success') == True, f"XMLHttpRequest failed: {js_result}"
-        # The result is wrapped in 'results' list
-        results = js_result.get('results', [])
-        assert len(results) > 0, "No results in JS execution"
-        xhr_result = results[0]
-        assert xhr_result.get('success') == True, f"XMLHttpRequest failed: {xhr_result}"
-        assert xhr_result.get('length', 0) > 0, f"No data received from XMLHttpRequest: {xhr_result}"
-
-
-@pytest.mark.asyncio
-async def test_browser_manager_without_cors_flag():
-    """Ensure that without --disable-web-security, normal functionality still works"""
-    browser_config = BrowserConfig(headless=True)
-
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        result = await crawler.arun(url="https://example.com", bypass_cache=True)
-        assert result.success
-        assert result.html
-
-
-# Entry point for debugging
-if __name__ == "__main__":
-    pytest.main([__file__, "-v"])