[Test EngSys] adding compression to scrubbers (Azure#18930)

* adding compression to scrubbers

* updates from pair programming with mccoy

* decompressed body stored as a string now

* moving subscription id replacing to a helper function, adding to response url

* restructuring

* adding six to utilities

* fixes based on charles/laurent

Author: seankane-msft

tools/azure-devtools/src/azure_devtools/scenario_tests/base.py

@@ -16,7 +16,7 @@
 
 from .config import TestConfig
 from .const import ENV_TEST_DIAGNOSE
-from .utilities import create_random_name
+from .utilities import create_random_name, _decompress_response_body
 from .decorators import live_only
 
 
@@ -186,6 +186,7 @@ def _process_response_recording(self, response):
             response['headers'] = headers
 
             body = response['body']['string']
+            response = _decompress_response_body(response)
             if is_text_payload(response) and body and not isinstance(body, six.string_types):
                 try:
                     response['body']['string'] = body.decode('utf-8')

tools/azure-devtools/src/azure_devtools/scenario_tests/recording_processors.py

@@ -3,9 +3,15 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 from copy import deepcopy
+from zlib import decompress
 import six
 
-from .utilities import is_text_payload, is_json_payload, is_batch_payload
+from .utilities import (
+    is_text_payload,
+    is_json_payload,
+    is_batch_payload,
+    replace_subscription_id
+)
 
 
 class RecordingProcessor(object):
@@ -36,42 +42,27 @@ def __init__(self, replacement):
         self._replacement = replacement
 
     def process_request(self, request):
-        request.uri = self._replace_subscription_id(request.uri)
+        request.uri = replace_subscription_id(request.uri, replacement=self._replacement)
 
         if is_text_payload(request) and request.body:
-            request.body = self._replace_subscription_id(request.body.decode()).encode()
+            request.body = replace_subscription_id(request.body.decode(), replacement=self._replacement).encode()
 
         return request
 
     def process_response(self, response):
         if is_text_payload(response) and response['body']['string']:
-            response['body']['string'] = self._replace_subscription_id(response['body']['string'])
+            response['body']['string'] = replace_subscription_id(response['body']['string'], replacement=self._replacement)
 
-        self.replace_header_fn(response, 'location', self._replace_subscription_id)
-        self.replace_header_fn(response, 'azure-asyncoperation', self._replace_subscription_id)
+        self.replace_header_fn(response, 'location', replace_subscription_id)
+        self.replace_header_fn(response, 'azure-asyncoperation', replace_subscription_id)
 
         try:
-            response["url"] = self._replace_subscription_id(response["url"])
+            response["url"] = replace_subscription_id(response["url"], replacement=self._replacement)
         except KeyError:
             pass
 
         return response
 
-    def _replace_subscription_id(self, val):
-        import re
-        # subscription presents in all api call
-        retval = re.sub('/(subscriptions)/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
-                        r'/\1/{}'.format(self._replacement),
-                        val,
-                        flags=re.IGNORECASE)
-
-        # subscription is also used in graph call
-        retval = re.sub('https://(graph.windows.net)/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
-                        r'https://\1/{}'.format(self._replacement),
-                        retval,
-                        flags=re.IGNORECASE)
-        return retval
-
 
 class LargeRequestBodyProcessor(RecordingProcessor):
     def __init__(self, max_request_body=128):
@@ -238,6 +229,8 @@ def process_response(self, response):
             self.replace_header(response, 'azure-asyncoperation', old, new)
             self.replace_header(response, "www-authenticate", old, new)
 
+        response["url"] = replace_subscription_id(response["url"])
+
         try:
             for old, new in self.names_name:
                 response["url"].replace(old, new)

tools/azure-devtools/src/azure_devtools/scenario_tests/utilities.py

@@ -3,11 +3,14 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
+import base64
 import hashlib
+import inspect
 import math
 import os
-import base64
-import inspect
+import re
+import six
+import zlib
 
 
 def create_random_name(prefix='aztest', length=24):
@@ -88,3 +91,31 @@ def trim_kwargs_from_test_function(fn, kwargs):
 
 def is_preparer_func(fn):
     return getattr(fn, '__is_preparer', False)
+
+
+def _decompress_response_body(response):
+    if "content-encoding" in response["headers"]:
+        enc = response['headers']['content-encoding'].lower()
+        if enc in ["gzip", "deflate"] and isinstance(response["body"]["string"], six.binary_type):
+            zlib_mode = 16 + zlib.MAX_WBITS if enc == "gzip" else zlib.MAX_WBITS
+            decompressor = zlib.decompressobj(wbits=zlib_mode)
+            decompressed = decompressor.decompress(response["body"]["string"])
+            decompressed = decompressed.decode("utf-8")
+            response["body"]["string"] = decompressed
+            response["headers"].pop("content-encoding")
+    return response
+
+
+def replace_subscription_id(val, replacement="00000000-0000-0000-0000-000000000000"):
+    # subscription presents in all api call
+    retval = re.sub('/(subscriptions)/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
+                    r'/\1/{}'.format(replacement),
+                    val,
+                    flags=re.IGNORECASE)
+
+    # subscription is also used in graph call
+    retval = re.sub('https://(graph.windows.net)/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
+                    r'https://\1/{}'.format(replacement),
+                    retval,
+                    flags=re.IGNORECASE)
+    return retval