diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py index 943867d0..76977bb9 100644 --- a/crawl4ai/async_crawler_strategy.py +++ b/crawl4ai/async_crawler_strategy.py @@ -1383,9 +1383,10 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy): try: await self.adapter.evaluate(page, f""" - (() => {{ + (async () => {{ try {{ - {remove_overlays_js} + const removeOverlays = {remove_overlays_js}; + await removeOverlays(); return {{ success: true }}; }} catch (error) {{ return {{ diff --git a/tests/general/test_async_crawler_strategy.py b/tests/general/test_async_crawler_strategy.py index 8426fe0a..e979325f 100644 --- a/tests/general/test_async_crawler_strategy.py +++ b/tests/general/test_async_crawler_strategy.py @@ -364,5 +364,19 @@ async def test_network_error_handling(): async with AsyncPlaywrightCrawlerStrategy() as strategy: await strategy.crawl("https://invalid.example.com", config) +@pytest.mark.asyncio +async def test_remove_overlay_elements(crawler_strategy): + config = CrawlerRunConfig( + remove_overlay_elements=True, + delay_before_return_html=5, + ) + + response = await crawler_strategy.crawl( + "https://www2.hm.com/en_us/index.html", + config + ) + assert response.status_code == 200 + assert "Accept all cookies" not in response.html + if __name__ == "__main__": pytest.main([__file__, "-v"]) \ No newline at end of file