Merge pull request #79 from starrify/more-stats-to-include

Added: More stats entries to include

Author: dangra

scrapy_crawlera/middleware.py

@@ -191,8 +191,13 @@ def process_response(self, request, response, spider):
         self._restore_original_delay(request)
 
         if self._is_no_available_proxies(response) or self._is_auth_error(response):
-            self._set_custom_delay(request, next(self.exp_backoff))
+            if self._is_no_available_proxies(response):
+                reason = 'noslaves'
+            else:
+                reason = 'autherror'
+            self._set_custom_delay(request, next(self.exp_backoff), reason=reason)
         else:
+            self.crawler.stats.inc_value('crawlera/delay/reset_backoff')
             self.exp_backoff = exp_backoff(self.backoff_step, self.backoff_max)
 
         if self._is_auth_error(response):
@@ -202,6 +207,7 @@ def process_response(self, request, response, spider):
             if retries < self.max_auth_retry_times:
                 return self._retry_auth(response, request, spider)
             else:
+                self.crawler.stats.inc_value('crawlera/retries/auth/max_reached')
                 logging.warning(
                     "Max retries for authentication issues reached, please check auth"
                     " information settings",
@@ -215,7 +221,7 @@ def process_response(self, request, response, spider):
             else:
                 after = response.headers.get('retry-after')
                 if after:
-                    self._set_custom_delay(request, float(after))
+                    self._set_custom_delay(request, float(after), reason='banned')
             self.crawler.stats.inc_value('crawlera/response/banned')
         else:
             self._bans[key] = 0
@@ -235,7 +241,7 @@ def process_exception(self, request, exception, spider):
         if isinstance(exception, (ConnectionRefusedError, ConnectionDone)):
             # Handle crawlera downtime
             self._clear_dns_cache()
-            self._set_custom_delay(request, self.connection_refused_delay)
+            self._set_custom_delay(request, self.connection_refused_delay, reason='conn_refused')
 
     def _handle_not_enabled_response(self, request, response):
         if self._should_enable_for_response(response):
@@ -244,6 +250,7 @@ def _handle_not_enabled_response(self, request, response):
 
             retryreq = request.copy()
             retryreq.dont_filter = True
+            self.crawler.stats.inc_value('crawlera/retries/should_have_been_enabled')
             return retryreq
         return response
 
@@ -256,6 +263,7 @@ def _retry_auth(self, response, request, spider):
         retryreq = request.copy()
         retryreq.meta['crawlera_auth_retry_times'] = retries
         retryreq.dont_filter = True
+        self.crawler.stats.inc_value('crawlera/retries/auth')
         return retryreq
 
     def _clear_dns_cache(self):
@@ -286,14 +294,17 @@ def _get_slot(self, request):
         key = self._get_slot_key(request)
         return key, self.crawler.engine.downloader.slots.get(key)
 
-    def _set_custom_delay(self, request, delay):
+    def _set_custom_delay(self, request, delay, reason=None):
         """Set custom delay for slot and save original one."""
         key, slot = self._get_slot(request)
         if not slot:
             return
         if self._saved_delays[key] is None:
             self._saved_delays[key] = slot.delay
         slot.delay = delay
+        if reason is not None:
+            self.crawler.stats.inc_value('crawlera/delay/%s' % reason)
+            self.crawler.stats.inc_value('crawlera/delay/%s/total' % reason, delay)
 
     def _restore_original_delay(self, request):
         """Restore original delay for slot if it was changed."""

tests/test_crawlera.py

@@ -702,7 +702,9 @@ def test_process_response_enables_crawlera(self):
         self.assertIsInstance(out, Request)
         self.assertEqual(mw.enabled, False)
         self.assertEqual(mw.enabled_for_domain["scrapy.org"], True)
-        self.assertEqual(mw.crawler.stats.get_stats(), {})
+        self.assertEqual(mw.crawler.stats.get_stats(), {
+            'crawlera/retries/should_have_been_enabled': 1,
+        })
 
         # Another regular response with bad code should be done on crawlera
         # and not be retried