Fix check is crawlera response (#89)

* Use empty values on headers randomly

* Check if the X-Crawlera-Version header is present

Author: denisgermano

scrapy_crawlera/middleware.py

@@ -283,7 +283,7 @@ def _get_url_domain(self, url):
         return parsed.netloc
 
     def _is_crawlera_response(self, response):
-        return bool(response.headers.get("X-Crawlera-Version"))
+        return bool("X-Crawlera-Version" in response.headers)
 
     def _get_slot_key(self, request):
         return request.meta.get('download_slot')

tests/test_crawlera.py

@@ -1,4 +1,5 @@
 import pytest
+from random import choice
 from unittest import TestCase
 try:
     from unittest.mock import call, patch
@@ -36,7 +37,8 @@ def setUp(self):
         self.settings = {'CRAWLERA_APIKEY': 'apikey'}
 
     def _mock_crawlera_response(self, url, headers=None, **kwargs):
-        crawlera_headers = {"X-Crawlera-Version": "1.36.3-cd5e44"}
+        crawlera_version = choice(("1.36.3-cd5e44", "", None))
+        crawlera_headers = {"X-Crawlera-Version": crawlera_version}
         if headers:
             crawlera_headers.update(headers)
         return Response(url, headers=crawlera_headers, **kwargs)