[ACR] Support set/get non-oci manifest (Azure#29552)

Author: Yalin Li

.vscode/cspell.json

@@ -610,7 +610,10 @@
     {
       "filename": "sdk/containerregistry/azure-containerregistry/**",
       "words": [
-        "udpated"
+        "udpated",
+        "cncf",
+        "oras",
+        "rootfs"
       ]
     },
     {

sdk/containerregistry/azure-containerregistry/CHANGELOG.md

@@ -1,15 +1,16 @@
 # Release History
 
-## 1.1.0b4 (Unreleased)
+## 1.1.0b4 (2023-04-25)
 
 ### Features Added
+- Added an optional kwarg `media_type` in `set_manifest()` to enable uploading image manifests of any type.
 
 ### Breaking Changes
-
-### Bugs Fixed
+- Renamed `upload_manifest()` to `set_manifest()`, and changed to consume manifest in `JSON` instead of `OCIManifest` type.
+- Renamed `download_manifest()` to `get_manifest()`, and changed it's return type from `DownloadManifestResult` to `GetManifestResult`.
 
 ### Other Changes
-- Changed the default audience to "https://containerregistry.azure.net" which works for all clouds.
+- Changed the default audience to `"https://containerregistry.azure.net"` which works for all clouds.
 
 ## 1.1.0b3 (2023-04-04)
 
@@ -65,7 +66,7 @@
 ### Features Added
 
 - Updated the supported rest api version to be the stable "2021-07-01".
-  - Removed the property `teleport_enabled` in `RepositoryProperties`.
+- Removed the property `teleport_enabled` in `RepositoryProperties`.
 
 ## 1.0.0b6 (2021-09-08)
 

sdk/containerregistry/azure-containerregistry/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/containerregistry/azure-containerregistry",
-  "Tag": "python/containerregistry/azure-containerregistry_fbbcf05c71"
+  "Tag": "python/containerregistry/azure-containerregistry_7f3bea3b69"
 }

sdk/containerregistry/azure-containerregistry/azure/containerregistry/__init__.py

@@ -17,7 +17,7 @@
     ArtifactManifestProperties,
     RepositoryProperties,
     ArtifactTagProperties,
-    DownloadManifestResult,
+    GetManifestResult,
 )
 from ._download_stream import DownloadBlobStream
 from ._version import VERSION
@@ -33,6 +33,6 @@
     "RepositoryProperties",
     "ArtifactTagOrder",
     "ArtifactTagProperties",
-    "DownloadManifestResult",
+    "GetManifestResult",
     "DownloadBlobStream",
 ]

sdk/containerregistry/azure-containerregistry/azure/containerregistry/_container_registry_client.py

@@ -6,8 +6,9 @@
 # pylint: disable=too-many-lines
 import functools
 import hashlib
+import json
 from io import BytesIO
-from typing import Any, Dict, IO, Optional, overload, Union, cast, Tuple
+from typing import Any, Dict, IO, Optional, overload, Union, cast, Tuple, MutableMapping
 
 from azure.core.credentials import TokenCredential
 from azure.core.exceptions import (
@@ -22,28 +23,30 @@
 from azure.core.tracing.decorator import distributed_trace
 
 from ._base_client import ContainerRegistryBaseClient
-from ._generated.models import AcrErrors, OCIManifest, ManifestWrapper
+from ._generated.models import AcrErrors
 from ._download_stream import DownloadBlobStream
 from ._helpers import (
     _compute_digest,
     _is_tag,
     _parse_next_link,
-    _serialize_manifest,
     _validate_digest,
-    OCI_MANIFEST_MEDIA_TYPE,
     SUPPORTED_API_VERSIONS,
+    OCI_IMAGE_MANIFEST,
+    SUPPORTED_MANIFEST_MEDIA_TYPES,
     DEFAULT_AUDIENCE,
     DEFAULT_CHUNK_SIZE,
 )
 from ._models import (
     RepositoryProperties,
     ArtifactTagProperties,
     ArtifactManifestProperties,
-    DownloadManifestResult,
+    GetManifestResult,
 )
 
-def _return_response_and_deserialized(pipeline_response, deserialized, _):
-    return pipeline_response, deserialized
+JSON = MutableMapping[str, Any]
+
+def _return_response(pipeline_response, _, __):
+    return pipeline_response
 
 def _return_response_and_headers(pipeline_response, _, response_headers):
     return pipeline_response, response_headers
@@ -859,24 +862,35 @@ def update_repository_properties(
         )
 
     @distributed_trace
-    def upload_manifest(
-        self, repository: str, manifest: Union[OCIManifest, IO], *, tag: Optional[str] = None, **kwargs
+    def set_manifest(
+        self,
+        repository: str,
+        manifest: Union[JSON, IO[bytes]],
+        *,
+        tag: Optional[str] = None,
+        media_type: str = OCI_IMAGE_MANIFEST,
+        **kwargs
     ) -> str:
-        """Upload a manifest for an OCI artifact.
+        """Set a manifest for an artifact.
 
-        :param str repository: Name of the repository.
-        :param manifest: The manifest to upload. Note: This must be a seekable stream.
-        :type manifest: ~azure.containerregistry.models.OCIManifest or IO
+        :param str repository: Name of the repository
+        :param manifest: The manifest to set. It can be a JSON formatted dict or seekable stream.
+        :type manifest: dict or IO
         :keyword tag: Tag of the manifest.
         :paramtype tag: str or None
-        :returns: The digest of the uploaded manifest, calculated by the registry.
+        :keyword media_type: The media type of the manifest. If not specified, this value will be set to
+            a default value of "application/vnd.oci.image.manifest.v1+json". Note: the current known media types are:
+            "application/vnd.oci.image.manifest.v1+json", and "application/vnd.docker.distribution.manifest.v2+json".
+        :paramtype media_type: str
+        :returns: The digest of the set manifest, calculated by the registry.
         :rtype: str
         :raises ValueError: If the parameter repository or manifest is None,
-            or the digest in the response does not match the digest of the uploaded manifest.
+            or the digest in the response does not match the digest of the set manifest.
         """
         try:
-            if isinstance(manifest, OCIManifest):
-                data = _serialize_manifest(manifest)
+            data: IO[bytes]
+            if isinstance(manifest, MutableMapping):
+                data = BytesIO(json.dumps(manifest).encode())
             else:
                 data = manifest
             tag_or_digest = tag
@@ -887,20 +901,55 @@ def upload_manifest(
                 name=repository,
                 reference=tag_or_digest,
                 payload=data,
-                content_type=OCI_MANIFEST_MEDIA_TYPE,
-                headers={"Accept": OCI_MANIFEST_MEDIA_TYPE},
+                content_type=media_type,
                 cls=_return_response_headers,
                 **kwargs
             )
             digest = response_headers['Docker-Content-Digest']
             if not _validate_digest(data, digest):
-                raise ValueError("The digest in the response does not match the digest of the uploaded manifest.")
+                raise ValueError("The server-computed digest does not match the client-computed digest.")
         except Exception as e:
             if repository is None or manifest is None:
                 raise ValueError("The parameter repository and manifest cannot be None.") from e
             raise
         return digest
 
+    @distributed_trace
+    def get_manifest(self, repository: str, tag_or_digest: str, **kwargs) -> GetManifestResult:
+        """Get the manifest for an artifact.
+
+        :param str repository: Name of the repository.
+        :param str tag_or_digest: The tag or digest of the manifest to get.
+            When digest is provided, will use this digest to compare with the one calculated by the response payload.
+            When tag is provided, will use the digest in response headers to compare.
+        :returns: GetManifestResult
+        :rtype: ~azure.containerregistry.GetManifestResult
+        :raises ValueError: If the requested digest does not match the digest of the received manifest.
+        """
+        response = cast(
+            PipelineResponse,
+            self._client.container_registry.get_manifest(
+                name=repository,
+                reference=tag_or_digest,
+                accept=SUPPORTED_MANIFEST_MEDIA_TYPES,
+                cls=_return_response,
+                **kwargs
+            )
+        )
+        media_type = response.http_response.headers['Content-Type']
+        manifest_bytes = response.http_response.read()
+        manifest_json = response.http_response.json()
+        if tag_or_digest.startswith("sha256:"):
+            digest = tag_or_digest
+            if not _validate_digest(manifest_bytes, digest):
+                raise ValueError("The requested digest does not match the digest of the received manifest.")
+        else:
+            digest = response.http_response.headers['Docker-Content-Digest']
+            if not _validate_digest(manifest_bytes, digest):
+                raise ValueError("The server-computed digest does not match the client-computed digest.")
+
+        return GetManifestResult(digest=digest, manifest=manifest_json, media_type=media_type)
+
     @distributed_trace
     def upload_blob(self, repository: str, data: IO[bytes], **kwargs) -> Tuple[str, int]:
         """Upload an artifact blob.
@@ -949,40 +998,7 @@ def _upload_blob_chunk(self, location: str, data: IO[bytes], **kwargs) -> Tuple[
             hasher.update(buffer)
             buffer = data.read(DEFAULT_CHUNK_SIZE)
             blob_size += len(buffer)
-        return "sha256:" + hasher.hexdigest(), location, blob_size
-
-    @distributed_trace
-    def download_manifest(self, repository: str, tag_or_digest: str, **kwargs) -> DownloadManifestResult:
-        """Download the manifest for an OCI artifact.
-
-        :param str repository: Name of the repository.
-        :param str tag_or_digest: The tag or digest of the manifest to download.
-            When digest is provided, will use this digest to compare with the one calculated by the response payload.
-            When tag is provided, will use the digest in response headers to compare.
-        :returns: DownloadManifestResult
-        :rtype: ~azure.containerregistry.DownloadManifestResult
-        :raises ValueError: If the requested digest does not match the digest of the received manifest.
-        """
-        response, manifest_wrapper = cast(
-            Tuple[PipelineResponse, ManifestWrapper],
-            self._client.container_registry.get_manifest(
-                name=repository,
-                reference=tag_or_digest,
-                headers={"Accept": OCI_MANIFEST_MEDIA_TYPE},
-                cls=_return_response_and_deserialized,
-                **kwargs
-            )
-        )
-        manifest = OCIManifest.deserialize(cast(ManifestWrapper, manifest_wrapper).serialize())
-        manifest_stream = _serialize_manifest(manifest)
-        if tag_or_digest.startswith("sha256:"):
-            digest = tag_or_digest
-        else:
-            digest = response.http_response.headers['Docker-Content-Digest']
-        if not _validate_digest(manifest_stream, digest):
-            raise ValueError("The requested digest does not match the digest of the received manifest.")
-
-        return DownloadManifestResult(digest=digest, data=manifest_stream, manifest=manifest)
+        return f"sha256:{hasher.hexdigest()}", location, blob_size
 
     @distributed_trace
     def download_blob(self, repository: str, digest: str, **kwargs) -> DownloadBlobStream:

sdk/containerregistry/azure-containerregistry/azure/containerregistry/_generated/aio/operations/_operations.py

@@ -124,16 +124,21 @@ async def check_docker_v2_support(self, **kwargs: Any) -> None:  # pylint: disab
             return cls(pipeline_response, None, {})
 
     @distributed_trace_async
-    async def get_manifest(self, name: str, reference: str, **kwargs: Any) -> _models.ManifestWrapper:
+    async def get_manifest(
+        self, name: str, reference: str, *, accept: Optional[str] = None, **kwargs: Any
+    ) -> AsyncIterator[bytes]:
         """Get the manifest identified by ``name`` and ``reference`` where ``reference`` can be a tag or
         digest.
 
         :param name: Name of the image (including the namespace). Required.
         :type name: str
         :param reference: A tag or a digest, pointing to a specific image. Required.
         :type reference: str
-        :return: ManifestWrapper
-        :rtype: ~container_registry.models.ManifestWrapper
+        :keyword accept: Accept header string delimited by comma. For example,
+         application/vnd.docker.distribution.manifest.v2+json. Default value is None.
+        :paramtype accept: str
+        :return: Async iterator of the response bytes
+        :rtype: AsyncIterator[bytes]
         :raises ~azure.core.exceptions.HttpResponseError:
         """
         error_map = {
@@ -147,11 +152,12 @@ async def get_manifest(self, name: str, reference: str, **kwargs: Any) -> _model
         _headers = kwargs.pop("headers", {}) or {}
         _params = kwargs.pop("params", {}) or {}
 
-        cls: ClsType[_models.ManifestWrapper] = kwargs.pop("cls", None)
+        cls: ClsType[AsyncIterator[bytes]] = kwargs.pop("cls", None)
 
         request = build_container_registry_get_manifest_request(
             name=name,
             reference=reference,
+            accept=accept,
             headers=_headers,
             params=_params,
         )
@@ -160,7 +166,7 @@ async def get_manifest(self, name: str, reference: str, **kwargs: Any) -> _model
         }
         request.url = self._client.format_url(request.url, **path_format_arguments)
 
-        _stream = False
+        _stream = True
         pipeline_response: PipelineResponse = await self._client._pipeline.run(  # pylint: disable=protected-access
             request, stream=_stream, **kwargs
         )
@@ -172,12 +178,12 @@ async def get_manifest(self, name: str, reference: str, **kwargs: Any) -> _model
             error = self._deserialize.failsafe_deserialize(_models.AcrErrors, pipeline_response)
             raise HttpResponseError(response=response, model=error)
 
-        deserialized = self._deserialize("ManifestWrapper", pipeline_response)
+        deserialized = response.iter_bytes()
 
         if cls:
-            return cls(pipeline_response, deserialized, {})
+            return cls(pipeline_response, deserialized, {})  # type: ignore
 
-        return deserialized
+        return deserialized  # type: ignore
 
     @distributed_trace_async
     async def create_manifest(self, name: str, reference: str, payload: IO, **kwargs: Any) -> Any:

sdk/containerregistry/azure-containerregistry/azure/containerregistry/_generated/models/_models.py

@@ -1081,50 +1081,49 @@ def __init__(
         self.annotations = annotations
 
 
-class OCIManifest(_serialization.Model):
+class OCIManifest(Manifest):
     """Returns the requested OCI Manifest file.
 
+    :ivar schema_version: Schema version.
+    :vartype schema_version: int
     :ivar config: V2 image config descriptor.
     :vartype config: ~container_registry.models.Descriptor
     :ivar layers: List of V2 image layer information.
     :vartype layers: list[~container_registry.models.Descriptor]
     :ivar annotations: Additional information provided through arbitrary metadata.
     :vartype annotations: ~container_registry.models.Annotations
-    :ivar schema_version: Schema version.
-    :vartype schema_version: int
     """
 
     _attribute_map = {
+        "schema_version": {"key": "schemaVersion", "type": "int"},
         "config": {"key": "config", "type": "Descriptor"},
         "layers": {"key": "layers", "type": "[Descriptor]"},
         "annotations": {"key": "annotations", "type": "Annotations"},
-        "schema_version": {"key": "schemaVersion", "type": "int"},
     }
 
     def __init__(
         self,
         *,
+        schema_version: Optional[int] = None,
         config: Optional["_models.Descriptor"] = None,
         layers: Optional[List["_models.Descriptor"]] = None,
         annotations: Optional["_models.Annotations"] = None,
-        schema_version: Optional[int] = None,
         **kwargs: Any
     ) -> None:
         """
+        :keyword schema_version: Schema version.
+        :paramtype schema_version: int
         :keyword config: V2 image config descriptor.
         :paramtype config: ~container_registry.models.Descriptor
         :keyword layers: List of V2 image layer information.
         :paramtype layers: list[~container_registry.models.Descriptor]
         :keyword annotations: Additional information provided through arbitrary metadata.
         :paramtype annotations: ~container_registry.models.Annotations
-        :keyword schema_version: Schema version.
-        :paramtype schema_version: int
         """
-        super().__init__(**kwargs)
+        super().__init__(schema_version=schema_version, **kwargs)
         self.config = config
         self.layers = layers
         self.annotations = annotations
-        self.schema_version = schema_version
 
 
 class Paths108HwamOauth2ExchangePostRequestbodyContentApplicationXWwwFormUrlencodedSchema(_serialization.Model):