BlobClient.OpenWriteAsync() (Azure#25623)

* initial OpenWrite add

* initial test port

Moved open write tests from BlockBlobClientTests to OpenWriteTestBase
and BlobBaseClientOpenWriteTests.
Hooked up client functionality through BlobBaseClientOpenWriteTests and
BlockBlobClientOpenWriteTests.
Moved recordings into correct folder to be picked up by framework.
Most tests ran in playback out of box. Some are failing due to recording
mismatches.
Next step to determine if tests need correcting or if rerecording is
fine.

* rerecorded some tests

* BlobClientTests

Fixed bug with scope names.
Extended open write tests for BlobClient.
Recorded tests.

* tests

* integrated clientside encryption

openwrite now works with clientside encryption
updated tests

* export api

* fixed merge bug

restored changes lost in merge.

* fixed operation name bug & rerecord

* recording service version update

* changelog

Co-authored-by: jschrepp-MSFT <41338290+jschrepp-MSFT@users.noreply.github.com>

Author: jaschrep-msft

sdk/storage/Azure.Storage.Blobs/CHANGELOG.md

@@ -3,6 +3,7 @@
 ## 12.11.0-beta.3 (Unreleased)
 - Added support for service version 2021-04-10.
 - Added support for BlobContainerClient.FilterBlobsByTag().
+- Added support for BlobContainerClient.OpenWriteAsync().
 - Fixed bug where BlobSasBuilder.SetPermissions(string rawPermissions) was not properly handling the Permanent Delete ('y') and set Immutability Policy ('i') permissions.
 
 ### Other Changes

sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs

@@ -9,6 +9,8 @@ public BlobClient(System.Uri blobUri, Azure.AzureSasCredential credential, Azure
         public BlobClient(System.Uri blobUri, Azure.Core.TokenCredential credential, Azure.Storage.Blobs.BlobClientOptions options = null) { }
         public BlobClient(System.Uri blobUri, Azure.Storage.Blobs.BlobClientOptions options = null) { }
         public BlobClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredential credential, Azure.Storage.Blobs.BlobClientOptions options = null) { }
+        public virtual System.IO.Stream OpenWrite(bool overwrite, Azure.Storage.Blobs.Models.BlobOpenWriteOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
+        public virtual System.Threading.Tasks.Task<System.IO.Stream> OpenWriteAsync(bool overwrite, Azure.Storage.Blobs.Models.BlobOpenWriteOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
         public virtual Azure.Response<Azure.Storage.Blobs.Models.BlobContentInfo> Upload(System.BinaryData content) { throw null; }
         public virtual Azure.Response<Azure.Storage.Blobs.Models.BlobContentInfo> Upload(System.BinaryData content, Azure.Storage.Blobs.Models.BlobUploadOptions options, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
         public virtual Azure.Response<Azure.Storage.Blobs.Models.BlobContentInfo> Upload(System.BinaryData content, bool overwrite = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
@@ -806,6 +808,17 @@ public BlobOpenReadOptions(bool allowModifications) { }
         public long Position { get { throw null; } set { } }
         public Azure.Storage.DownloadTransactionalHashingOptions TransactionalHashingOptions { get { throw null; } set { } }
     }
+    public partial class BlobOpenWriteOptions
+    {
+        public BlobOpenWriteOptions() { }
+        public long? BufferSize { get { throw null; } set { } }
+        public Azure.Storage.Blobs.Models.BlobHttpHeaders HttpHeaders { get { throw null; } set { } }
+        public System.Collections.Generic.IDictionary<string, string> Metadata { get { throw null; } set { } }
+        public Azure.Storage.Blobs.Models.BlobRequestConditions OpenConditions { get { throw null; } set { } }
+        public System.IProgress<long> ProgressHandler { get { throw null; } set { } }
+        public System.Collections.Generic.IDictionary<string, string> Tags { get { throw null; } set { } }
+        public Azure.Storage.UploadTransactionalHashingOptions TransactionalHashingOptions { get { throw null; } set { } }
+    }
     public partial class BlobProperties
     {
         public BlobProperties() { }

sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs

@@ -1721,6 +1721,102 @@ internal async Task<Response<BlobContentInfo>> StagedUploadInternal(
         }
         #endregion Upload
 
+        #region OpenWrite
+        /// <summary>
+        /// Opens a stream for writing to the blob.
+        /// </summary>
+        /// <param name="overwrite">
+        /// Whether an existing blob should be deleted and recreated.
+        /// </param>
+        /// <param name="options">
+        /// Optional parameters.
+        /// </param>
+        /// <param name="cancellationToken">
+        /// Optional <see cref="CancellationToken"/> to propagate
+        /// notifications that the operation should be cancelled.
+        /// </param>
+        /// <returns>
+        /// A stream to write to the Append Blob.
+        /// </returns>
+        /// <remarks>
+        /// A <see cref="RequestFailedException"/> will be thrown if
+        /// a failure occurs.
+        /// </remarks>
+#pragma warning disable AZC0015 // Unexpected client method return type.
+        public virtual Stream OpenWrite(
+#pragma warning restore AZC0015 // Unexpected client method return type.
+            bool overwrite,
+            BlobOpenWriteOptions options = default,
+            CancellationToken cancellationToken = default)
+            => OpenWriteInternal(
+                overwrite,
+                options,
+                async: false,
+                cancellationToken).EnsureCompleted();
+
+        /// <summary>
+        /// Opens a stream for writing to the blob.  If the blob exists,
+        /// it will be overwritten.
+        /// </summary>
+        /// <param name="overwrite">
+        /// Whether an existing blob should be deleted and recreated.
+        /// </param>
+        /// <param name="options">
+        /// Optional parameters.
+        /// </param>
+        /// <param name="cancellationToken">
+        /// Optional <see cref="CancellationToken"/> to propagate
+        /// notifications that the operation should be cancelled.
+        /// </param>
+        /// <returns>
+        /// A stream to write to the Append Blob.
+        /// </returns>
+        /// <remarks>
+        /// A <see cref="RequestFailedException"/> will be thrown if
+        /// a failure occurs.
+        /// </remarks>
+#pragma warning disable AZC0015 // Unexpected client method return type.
+        public virtual async Task<Stream> OpenWriteAsync(
+#pragma warning restore AZC0015 // Unexpected client method return type.
+            bool overwrite,
+            BlobOpenWriteOptions options = default,
+            CancellationToken cancellationToken = default)
+            => await OpenWriteInternal(
+                overwrite,
+                options,
+                async: true,
+                cancellationToken).ConfigureAwait(false);
+
+        internal async Task<Stream> OpenWriteInternal(
+            bool overwrite,
+            BlobOpenWriteOptions options,
+            bool async,
+            CancellationToken cancellationToken)
+        {
+            if (UsingClientSideEncryption)
+            {
+                if (UsingClientSideEncryption && options.TransactionalHashingOptions != default)
+                {
+                    throw Errors.TransactionalHashingNotSupportedWithClientSideEncryption();
+                }
+
+                return await new BlobClientSideEncryptor(new ClientSideEncryptor(ClientSideEncryption))
+                    .ClientSideEncryptionOpenWriteInternal(
+                        BlockBlobClient,
+                        overwrite,
+                        options.ToBlockBlobOpenWriteOptions(),
+                        async,
+                        cancellationToken).ConfigureAwait(false);
+            }
+
+            return await BlockBlobClient.OpenWriteInternal(
+                overwrite,
+                options.ToBlockBlobOpenWriteOptions(),
+                async,
+                cancellationToken).ConfigureAwait(false);
+        }
+        #endregion
+
         private BlockBlobClient _blockBlobClient;
 
         private BlockBlobClient BlockBlobClient

sdk/storage/Azure.Storage.Blobs/src/BlobClientSideEncryptor.cs

@@ -6,6 +6,8 @@
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
+using Azure.Storage.Blobs.Models;
+using Azure.Storage.Blobs.Specialized;
 using Azure.Storage.Cryptography;
 using Azure.Storage.Cryptography.Models;
 using Metadata = System.Collections.Generic.IDictionary<string, string>;
@@ -48,12 +50,70 @@ public BlobClientSideEncryptor(ClientSideEncryptor encryptor)
                 async,
                 cancellationToken).ConfigureAwait(false);
 
+            Metadata modifiedMetadata = TransformMetadata(metadata, EncryptionData);
+
+            return (NonSeekableCiphertext, modifiedMetadata);
+        }
+
+        /// <summary>
+        /// Creates a crypto transform stream to write blob contents to.
+        /// </summary>
+        /// <param name="blobClient">
+        /// BlobClient to open write with.
+        /// </param>
+        /// <param name="overwrite">
+        /// Overwrite parameter to open write.
+        /// </param>
+        /// <param name="options">
+        /// Options parameter to open write.
+        /// </param>
+        /// <param name="async">
+        /// Whether to perform this operation asynchronously.
+        /// </param>
+        /// <param name="cancellationToken">
+        /// Cancellation token.
+        /// </param>
+        /// <returns>
+        /// Content transform write stream and metadata.
+        /// </returns>
+        public async Task<Stream> ClientSideEncryptionOpenWriteInternal(
+            BlockBlobClient blobClient,
+            bool overwrite,
+            BlockBlobOpenWriteOptions options,
+            bool async,
+            CancellationToken cancellationToken)
+        {
+            Stream encryptionWriteStream = await _encryptor.EncryptedOpenWriteInternal(
+                async (encryptiondata, funcAsync, funcCancellationToken) =>
+                {
+                    options.Metadata = TransformMetadata(options.Metadata, encryptiondata);
+
+                    return await blobClient.OpenWriteInternal(
+                            overwrite,
+                            options,
+                            funcAsync,
+                            funcCancellationToken).ConfigureAwait(false);
+                },
+                async,
+                cancellationToken).ConfigureAwait(false);
+
+            return encryptionWriteStream;
+        }
+
+        /// <summary>
+        /// Adds encryption data to provided blob metadata, overwriting previous entry if any.
+        /// Safely creates new metadata object if none is provided.
+        /// </summary>
+        /// <param name="metadata">Optionally existing metadata.</param>
+        /// <param name="encryptionData">Encryption data to add.</param>
+        private static Metadata TransformMetadata(Metadata metadata, EncryptionData encryptionData)
+        {
             Metadata modifiedMetadata = metadata == default
                 ? new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
                 : new Dictionary<string, string>(metadata, StringComparer.OrdinalIgnoreCase);
-            modifiedMetadata[Constants.ClientSideEncryption.EncryptionDataKey] = EncryptionDataSerializer.Serialize(EncryptionData);
+            modifiedMetadata[Constants.ClientSideEncryption.EncryptionDataKey] = EncryptionDataSerializer.Serialize(encryptionData);
 
-            return (NonSeekableCiphertext, modifiedMetadata);
+            return modifiedMetadata;
         }
     }
 }

sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs

@@ -2742,13 +2742,13 @@ public virtual async Task<Stream> OpenWriteAsync(
         /// A <see cref="RequestFailedException"/> will be thrown if
         /// a failure occurs.
         /// </remarks>
-        private async Task<Stream> OpenWriteInternal(
+        internal async Task<Stream> OpenWriteInternal(
             bool overwrite,
             BlockBlobOpenWriteOptions options,
             bool async,
             CancellationToken cancellationToken)
         {
-            string operationName = $"{nameof(BlockBlobClient)}.{nameof(OpenWrite)}";
+            string operationName = options?.OperationName ?? $"{nameof(BlockBlobClient)}.{nameof(OpenWrite)}";
             DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope(operationName);
 
             try

sdk/storage/Azure.Storage.Blobs/src/Models/BlobOpenWriteOptions.cs

@@ -0,0 +1,76 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using Azure.Storage.Blobs.Specialized;
+using Metadata = System.Collections.Generic.IDictionary<string, string>;
+using Tags = System.Collections.Generic.IDictionary<string, string>;
+
+#pragma warning disable SA1402  // File may only contain a single type
+
+namespace Azure.Storage.Blobs.Models
+{
+    /// <summary>
+    /// Optional parameters for Page Blob Open Write.
+    /// </summary>
+    public class BlobOpenWriteOptions
+    {
+        /// <summary>
+        /// The size of the buffer to use.  Default is 4 MB,
+        /// max is 4000 MB.  See <see cref="BlockBlobClient.BlockBlobMaxStageBlockLongBytes"/>.
+        /// </summary>
+        public long? BufferSize { get; set; }
+
+        /// <summary>
+        /// Access conditions used to open the write stream.
+        /// </summary>
+        public BlobRequestConditions OpenConditions { get; set; }
+
+        /// <summary>
+        /// Optional <see cref="IProgress{Long}"/> to provide
+        /// progress updates about data transfers.
+        /// </summary>
+        public IProgress<long> ProgressHandler { get; set; }
+
+        /// <summary>
+        /// Optional standard HTTP header properties that can be set for this block blob.
+        /// </summary>
+        public BlobHttpHeaders HttpHeaders { get; set; }
+
+        /// <summary>
+        /// Optional custom metadata to set for this block blob.
+        /// </summary>
+#pragma warning disable CA2227 // Collection properties should be readonly
+        public Metadata Metadata { get; set; }
+#pragma warning restore CA2227 // Collection properties should be readonly
+
+        /// <summary>
+        /// Options tags to set for this block blob.
+        /// </summary>
+#pragma warning disable CA2227 // Collection properties should be readonly
+        public Tags Tags { get; set; }
+#pragma warning restore CA2227 // Collection properties should be readonly
+
+        /// <summary>
+        /// Optional <see cref="UploadTransactionalHashingOptions"/> for using transactional
+        /// hashing on uploads.
+        /// </summary>
+        public UploadTransactionalHashingOptions TransactionalHashingOptions { get; set; }
+    }
+
+    internal static class BlobOpenWriteOptionsExtensions
+    {
+        public static BlockBlobOpenWriteOptions ToBlockBlobOpenWriteOptions(this BlobOpenWriteOptions options)
+            => new BlockBlobOpenWriteOptions
+            {
+                BufferSize = options.BufferSize,
+                OpenConditions = options.OpenConditions,
+                ProgressHandler = options.ProgressHandler,
+                HttpHeaders = options.HttpHeaders,
+                Metadata = options.Metadata,
+                Tags = options.Tags,
+                TransactionalHashingOptions = options.TransactionalHashingOptions,
+                OperationName = $"{nameof(BlobClient)}.{nameof(BlockBlobClient.OpenWrite)}"
+            };
+    }
+}

sdk/storage/Azure.Storage.Blobs/src/Models/BlockBlobOpenWriteOptions.cs

@@ -54,5 +54,7 @@ public class BlockBlobOpenWriteOptions
         /// hashing on uploads.
         /// </summary>
         public UploadTransactionalHashingOptions TransactionalHashingOptions { get; set; }
+
+        internal string OperationName { get; set; }
     }
 }