fix: Add CDP endpoint verification with exponential backoff for managed browsers (unclecode#1445)

browser_manager:
- Add CDP endpoint verification with retry logic and exponential backoff
- Call verification before connecting to CDP in `start()` method
- Graceful handling of timing issues during browser startup

test_cdp_strategy:
- Fix cookie persistence test by adding storage state management
- Fix session management test to work with managed browser architecture
- Add comprehensive CDP timing tests covering:
  - Fast startup scenarios
  - Delayed browser startup simulation
  - Exponential backoff behavior validation
  - Concurrent browser connections
  - Stress testing with multiple successive startups
  - Retry count verification

Impact:
- Eliminates browser startup failures due to CDP timing issues
- Provides robust fallback with automatic retries
- Maintains fast startup when CDP is immediately available
- Comprehensive test coverage ensures reliability

Resolves CDP connection timing issues in managed browser mode.

Author: SohamKukreti

crawl4ai/browser_manager.py

@@ -658,6 +658,11 @@ async def start(self):
         if self.config.cdp_url or self.config.use_managed_browser:
             self.config.use_managed_browser = True
             cdp_url = await self.managed_browser.start() if not self.config.cdp_url else self.config.cdp_url
+            
+            # Add CDP endpoint verification before connecting
+            if not await self._verify_cdp_ready(cdp_url):
+                raise Exception(f"CDP endpoint at {cdp_url} is not ready after startup")
+            
             self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url)
             contexts = self.browser.contexts
             if contexts:
@@ -678,6 +683,24 @@ async def start(self):
 
             self.default_context = self.browser
 
+    async def _verify_cdp_ready(self, cdp_url: str) -> bool:
+        """Verify CDP endpoint is ready with exponential backoff"""
+        import aiohttp
+        self.logger.debug(f"Starting CDP verification for {cdp_url}", tag="BROWSER")
+        for attempt in range(5):
+            try:
+                async with aiohttp.ClientSession() as session:
+                    async with session.get(f"{cdp_url}/json/version", timeout=aiohttp.ClientTimeout(total=2)) as response:
+                        if response.status == 200:
+                            self.logger.debug(f"CDP endpoint ready after {attempt + 1} attempts", tag="BROWSER")
+                            return True
+            except Exception as e:
+                self.logger.debug(f"CDP check attempt {attempt + 1} failed: {e}", tag="BROWSER")
+            delay = 0.5 * (1.4 ** attempt)
+            self.logger.debug(f"Waiting {delay:.2f}s before next CDP check...", tag="BROWSER")
+            await asyncio.sleep(delay)
+        self.logger.debug(f"CDP verification failed after 5 attempts", tag="BROWSER")
+        return False
 
     def _build_browser_args(self) -> dict:
         """Build browser launch arguments from config."""