[rest] unify response headers behaviors across transports (Azure#20234)

Author: iscai-msft

sdk/core/azure-core/CHANGELOG.md

@@ -15,6 +15,8 @@ an `encoding` parameter.
 
 ### Bugs Fixed
 
+- The behaviour of the headers returned in `azure.core.rest` responses now aligns across sync and async. Items can now be checked case-insensitively and without raising an error for format.
+
 ### Other Changes
 
 ## 1.17.0 (2021-08-05)

sdk/core/azure-core/azure/core/rest/_aiohttp.py

@@ -23,14 +23,97 @@
 # IN THE SOFTWARE.
 #
 # --------------------------------------------------------------------------
-
+import collections.abc
 import asyncio
+from itertools import groupby
 from typing import AsyncIterator
 from multidict import CIMultiDict
 from . import HttpRequest, AsyncHttpResponse
 from ._helpers_py3 import iter_raw_helper, iter_bytes_helper
 from ..pipeline.transport._aiohttp import AioHttpStreamDownloadGenerator
 
+class _ItemsView(collections.abc.ItemsView):
+    def __init__(self, ref):
+        super().__init__(ref)
+        self._ref = ref
+
+    def __iter__(self):
+        for key, groups in groupby(self._ref.__iter__(), lambda x: x[0]):
+            yield tuple([key, ", ".join(group[1] for group in groups)])
+
+    def __contains__(self, item):
+        if not (isinstance(item, (list, tuple)) and len(item) == 2):
+            return False
+        for k, v in self.__iter__():
+            if item[0].lower() == k.lower() and item[1] == v:
+                return True
+        return False
+
+    def __repr__(self):
+        return f"dict_items({list(self.__iter__())})"
+
+class _KeysView(collections.abc.KeysView):
+    def __init__(self, items):
+        super().__init__(items)
+        self._items = items
+
+    def __iter__(self):
+        for key, _ in self._items:
+            yield key
+
+    def __contains__(self, key):
+        for k in self.__iter__():
+            if key.lower() == k.lower():
+                return True
+        return False
+    def __repr__(self):
+        return f"dict_keys({list(self.__iter__())})"
+
+class _ValuesView(collections.abc.ValuesView):
+    def __init__(self, items):
+        super().__init__(items)
+        self._items = items
+
+    def __iter__(self):
+        for _, value in self._items:
+            yield value
+
+    def __contains__(self, value):
+        for v in self.__iter__():
+            if value == v:
+                return True
+        return False
+
+    def __repr__(self):
+        return f"dict_values({list(self.__iter__())})"
+
+
+class _CIMultiDict(CIMultiDict):
+    """Dictionary with the support for duplicate case-insensitive keys."""
+
+    def __iter__(self):
+        return iter(self.keys())
+
+    def keys(self):
+        """Return a new view of the dictionary's keys."""
+        return _KeysView(self.items())
+
+    def items(self):
+        """Return a new view of the dictionary's items."""
+        return _ItemsView(super().items())
+
+    def values(self):
+        """Return a new view of the dictionary's values."""
+        return _ValuesView(self.items())
+
+    def __getitem__(self, key: str) -> str:
+        return ", ".join(self.getall(key, []))
+
+    def get(self, key, default=None):
+        values = self.getall(key, None)
+        if values:
+            values = ", ".join(values)
+        return values or default
 
 class RestAioHttpTransportResponse(AsyncHttpResponse):
     def __init__(
@@ -41,7 +124,7 @@ def __init__(
     ):
         super().__init__(request=request, internal_response=internal_response)
         self.status_code = internal_response.status
-        self.headers = CIMultiDict(internal_response.headers)  # type: ignore
+        self.headers = _CIMultiDict(internal_response.headers)  # type: ignore
         self.reason = internal_response.reason
         self.content_type = internal_response.headers.get('content-type')
 

sdk/core/azure-core/azure/core/rest/_requests_basic.py

@@ -23,12 +23,42 @@
 # IN THE SOFTWARE.
 #
 # --------------------------------------------------------------------------
+try:
+    import collections.abc as collections
+except ImportError:
+    import collections  # type: ignore
+
 from typing import TYPE_CHECKING, cast
+from requests.structures import CaseInsensitiveDict
 
 from ..exceptions import ResponseNotReadError, StreamConsumedError, StreamClosedError
 from ._rest import _HttpResponseBase, HttpResponse
 from ..pipeline.transport._requests_basic import StreamDownloadGenerator
 
+class _ItemsView(collections.ItemsView):
+
+    def __contains__(self, item):
+        if not (isinstance(item, (list, tuple)) and len(item) == 2):
+            return False  # requests raises here, we just return False
+        for k, v in self.__iter__():
+            if item[0].lower() == k.lower() and item[1] == v:
+                return True
+        return False
+
+    def __repr__(self):
+        return 'ItemsView({})'.format(dict(self.__iter__()))
+
+class _CaseInsensitiveDict(CaseInsensitiveDict):
+    """Overriding default requests dict so we can unify
+    to not raise if users pass in incorrect items to contains.
+    Instead, we return False
+    """
+
+    def items(self):
+        """Return a new view of the dictionary's items."""
+        return _ItemsView(self)
+
+
 if TYPE_CHECKING:
     from typing import Iterator, Optional
 
@@ -43,7 +73,7 @@ class _RestRequestsTransportResponseBase(_HttpResponseBase):
     def __init__(self, **kwargs):
         super(_RestRequestsTransportResponseBase, self).__init__(**kwargs)
         self.status_code = self._internal_response.status_code
-        self.headers = self._internal_response.headers
+        self.headers = _CaseInsensitiveDict(self._internal_response.headers)
         self.reason = self._internal_response.reason
         self.content_type = self._internal_response.headers.get('content-type')
 

sdk/core/azure-core/azure/core/rest/_rest.py

@@ -306,7 +306,9 @@ class HttpResponse(_HttpResponseBase):  # pylint: disable=too-many-instance-attr
     :keyword request: The request that resulted in this response.
     :paramtype request: ~azure.core.rest.HttpRequest
     :ivar int status_code: The status code of this response
-    :ivar mapping headers: The response headers
+    :ivar mapping headers: The case-insensitive response headers.
+     While looking up headers is case-insensitive, when looking up
+     keys in `header.keys()`, we recommend using lowercase.
     :ivar str reason: The reason phrase for this response
     :ivar bytes content: The response content in bytes.
     :ivar str url: The URL that resulted in this response

sdk/core/azure-core/azure/core/rest/_rest_py3.py

@@ -227,7 +227,7 @@ def __init__(
         self.request = request
         self._internal_response = kwargs.pop("internal_response")
         self.status_code = None
-        self.headers = {}  # type: HeadersType
+        self.headers = _case_insensitive_dict({})
         self.reason = None
         self.is_closed = False
         self.is_stream_consumed = False
@@ -284,7 +284,7 @@ def json(self) -> Any:
         """
         # this will trigger errors if response is not read in
         self.content  # pylint: disable=pointless-statement
-        if not self._json:
+        if self._json is None:
             self._json = loads(self.text())
         return self._json
 
@@ -320,7 +320,9 @@ class HttpResponse(_HttpResponseBase):
     :keyword request: The request that resulted in this response.
     :paramtype request: ~azure.core.rest.HttpRequest
     :ivar int status_code: The status code of this response
-    :ivar mapping headers: The response headers
+    :ivar mapping headers: The case-insensitive response headers.
+     While looking up headers is case-insensitive, when looking up
+     keys in `header.keys()`, we recommend using lowercase.
     :ivar str reason: The reason phrase for this response
     :ivar bytes content: The response content in bytes.
     :ivar str url: The URL that resulted in this response