Merge pull request unclecode#1528 from unclecode/fix/managed-browser-cdp-timing

Add CDP endpoint verification with exponential backoff for managed browsers

Author: ntohidi

crawl4ai/browser_manager.py

@@ -661,6 +661,11 @@ async def start(self):
         if self.config.cdp_url or self.config.use_managed_browser:
             self.config.use_managed_browser = True
             cdp_url = await self.managed_browser.start() if not self.config.cdp_url else self.config.cdp_url
+            
+            # Add CDP endpoint verification before connecting
+            if not await self._verify_cdp_ready(cdp_url):
+                raise Exception(f"CDP endpoint at {cdp_url} is not ready after startup")
+            
             self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url)
             contexts = self.browser.contexts
             if contexts:
@@ -681,6 +686,24 @@ async def start(self):
 
             self.default_context = self.browser
 
+    async def _verify_cdp_ready(self, cdp_url: str) -> bool:
+        """Verify CDP endpoint is ready with exponential backoff"""
+        import aiohttp
+        self.logger.debug(f"Starting CDP verification for {cdp_url}", tag="BROWSER")
+        for attempt in range(5):
+            try:
+                async with aiohttp.ClientSession() as session:
+                    async with session.get(f"{cdp_url}/json/version", timeout=aiohttp.ClientTimeout(total=2)) as response:
+                        if response.status == 200:
+                            self.logger.debug(f"CDP endpoint ready after {attempt + 1} attempts", tag="BROWSER")
+                            return True
+            except Exception as e:
+                self.logger.debug(f"CDP check attempt {attempt + 1} failed: {e}", tag="BROWSER")
+            delay = 0.5 * (1.4 ** attempt)
+            self.logger.debug(f"Waiting {delay:.2f}s before next CDP check...", tag="BROWSER")
+            await asyncio.sleep(delay)
+        self.logger.debug(f"CDP verification failed after 5 attempts", tag="BROWSER")
+        return False
 
     def _build_browser_args(self) -> dict:
         """Build browser launch arguments from config."""