Merge pull request unclecode#1529 from unclecode/fix/remove_overlay_elements

Fix remove_overlay_elements functionality by calling injected JS function.

Author: ntohidi

crawl4ai/async_crawler_strategy.py

@@ -1383,9 +1383,10 @@ async def remove_overlay_elements(self, page: Page) -> None:
         try:
             await self.adapter.evaluate(page,
                 f"""
-                (() => {{
+                (async () => {{
                     try {{
-                        {remove_overlays_js}
+                        const removeOverlays = {remove_overlays_js};
+                        await removeOverlays();
                         return {{ success: true }};
                     }} catch (error) {{
                         return {{

tests/general/test_async_crawler_strategy.py

@@ -364,5 +364,19 @@ async def test_network_error_handling():
         async with AsyncPlaywrightCrawlerStrategy() as strategy:
             await strategy.crawl("https://invalid.example.com", config)
 
+@pytest.mark.asyncio
+async def test_remove_overlay_elements(crawler_strategy):
+    config = CrawlerRunConfig(
+        remove_overlay_elements=True,
+        delay_before_return_html=5,
+    )
+    
+    response = await crawler_strategy.crawl(
+        "https://www2.hm.com/en_us/index.html",
+        config
+    )
+    assert response.status_code == 200
+    assert "Accept all cookies" not in response.html
+
 if __name__ == "__main__":
     pytest.main([__file__, "-v"])