[DataLake][RecursiveAcl]Adopt AzureError change (Azure#14906)

* [DataLake][RecursiveAcl]Adopt AzureError change

* add sample

* address comments

* parse StorageErrorException

* use latest azure-core 1.9.0 as min-dependency

* rename progress_callback in sample to progress_hook

* update shared dependency

* add python 3.9 as classifier

Author: xiafu-msft

sdk/storage/azure-storage-blob/CHANGELOG.md

@@ -1,7 +1,17 @@
 # Release History
 
-## 12.6.0b2 (Unreleased)
+## 12.6.0 (2020-11-10)
+**Stable release of preview features**
+- Preview feature `ArrowDialect` as output format of `query_blob`
+- Preview feature `undelete_container` on BlobServiceClient.
+- Preview feature Last Access Time.
 
+**Fixes**
+- Fixed the expired Authorization token problem during retry (#14701, #14067)
+- Catch exceptions thrown by async download (#14319)
+
+**Notes**
+- Updated dependency `azure-core` from  azure-core<2.0.0,>=1.6.0 to azure-core<2.0.0,>=1.9.0 to get continuation_token attr on AzureError.
 
 ## 12.6.0b1 (2020-10-02)
 **New features***

sdk/storage/azure-storage-blob/azure/storage/blob/_version.py

@@ -4,4 +4,4 @@
 # license information.
 # --------------------------------------------------------------------------
 
-VERSION = "12.6.0b2"
+VERSION = "12.6.0"

sdk/storage/azure-storage-blob/setup.py

@@ -71,7 +71,7 @@
     author_email='ascl@microsoft.com',
     url='https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/storage/azure-storage-blob',
     classifiers=[
-        "Development Status :: 4 - Beta",
+        "Development Status :: 5 - Production/Stable",
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
@@ -80,6 +80,7 @@
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'License :: OSI Approved :: MIT License',
     ],
     zip_safe=False,
@@ -92,7 +93,7 @@
         'tests.common'
     ]),
     install_requires=[
-        "azure-core<2.0.0,>=1.6.0",
+        "azure-core<2.0.0,>=1.9.0",
         "msrest>=0.6.10",
         "cryptography>=2.1.4"
     ],

sdk/storage/azure-storage-file-datalake/CHANGELOG.md

@@ -1,6 +1,15 @@
 # Release History
-## 12.2.0b2 (Unreleased)
+## 12.2.0 (2020-11-10)
+**Stable release of preview features**
+- Preview feature set/update/remove access control recursively.
+- Preview feature `set_file_expiry` on DataLakeFileClient.
+- Preview feature generating directory level sas.
 
+**Fixes**
+- Fixed session closure of filesystem (#14497)
+
+**Notes**
+- Updated dependency `azure-core` from  azure-core<2.0.0,>=1.6.0 to azure-core<2.0.0,>=1.9.0
 
 ## 12.2.0b1 (2020-10-02)
 **New Features**

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/__init__.py

@@ -32,7 +32,6 @@
     ArrowDialect,
     ArrowType,
     DataLakeFileQueryError,
-    DataLakeAclChangeFailedError,
     AccessControlChangeResult,
     AccessControlChangeCounters,
     AccessControlChangeFailure,
@@ -86,7 +85,6 @@
     'DelimitedTextDialect',
     'DelimitedJsonDialect',
     'DataLakeFileQueryError',
-    'DataLakeAclChangeFailedError',
     'ArrowDialect',
     'ArrowType',
     'DataLakeFileQueryError'

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_deserialize.py

@@ -140,7 +140,8 @@ def process_storage_error(storage_error):
     for name, info in additional_data.items():
         error_message += "\n{}:{}".format(name, info)
 
-    error = raise_error(message=error_message, response=storage_error.response)
+    error = raise_error(message=error_message, response=storage_error.response,
+                        continuation_token=storage_error.continuation_token)
     error.error_code = error_code
     error.additional_info = additional_data
     raise error

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_models.py

@@ -818,20 +818,3 @@ def __init__(self, batch_counters, aggregate_counters, batch_failures, continuat
         self.aggregate_counters = aggregate_counters
         self.batch_failures = batch_failures
         self.continuation = continuation
-
-
-class DataLakeAclChangeFailedError(Exception):
-    """The error happened during set/update/remove acl recursive operation.
-
-    :ivar ~azure.core.exceptions.AzureError error:
-        The exception.
-    :ivar str description:
-        A description of the error.
-    :ivar str continuation:
-        An opaque continuation token that may be used to resume the operations in case of failures.
-    """
-
-    def __init__(self, error, description, continuation):
-        self.error = error
-        self.description = description
-        self.continuation = continuation

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_path_client.py

@@ -19,7 +19,7 @@
 from ._generated import DataLakeStorageClient
 from ._generated.models import StorageErrorException
 from ._models import LocationMode, DirectoryProperties, AccessControlChangeResult, AccessControlChanges, \
-    AccessControlChangeCounters, AccessControlChangeFailure, DataLakeAclChangeFailedError
+    AccessControlChangeCounters, AccessControlChangeFailure
 from ._serialize import convert_dfs_url_to_blob_url, get_mod_conditions, \
     get_path_http_headers, add_metadata_headers, get_lease_id, get_source_mod_conditions, get_access_conditions
 from ._shared.base_client import StorageAccountHostsMixin, parse_query
@@ -405,7 +405,7 @@ def _set_access_control_recursive_options(mode, acl, **kwargs):
             'mode': mode,
             'force_flag': kwargs.pop('continue_on_failure', None),
             'timeout': kwargs.pop('timeout', None),
-            'continuation': kwargs.pop('continuation', None),
+            'continuation': kwargs.pop('continuation_token', None),
             'max_records': kwargs.pop('batch_size', None),
             'acl': acl,
             'cls': return_headers_and_deserialized}
@@ -429,7 +429,7 @@ def set_access_control_recursive(self,
         :keyword func(~azure.storage.filedatalake.AccessControlChanges) progress_hook:
             Callback where the caller can track progress of the operation
             as well as collect paths that failed to change Access Control.
-        :keyword str continuation:
+        :keyword str continuation_token:
             Optional continuation token that can be used to resume previously stopped operation.
         :keyword int batch_size:
             Optional. If data set size exceeds batch size then operation will be split into multiple
@@ -451,6 +451,8 @@ def set_access_control_recursive(self,
         :return: A summary of the recursive operations, including the count of successes and failures,
             as well as a continuation token in case the operation was terminated prematurely.
         :rtype: :class:`~azure.storage.filedatalake.AccessControlChangeResult`
+        :raises ~azure.core.exceptions.AzureError:
+            User can restart the operation using continuation_token field of AzureError if the token is available.
         """
         if not acl:
             raise ValueError("The Access Control List must be set for this operation")
@@ -478,7 +480,7 @@ def update_access_control_recursive(self,
         :keyword func(~azure.storage.filedatalake.AccessControlChanges) progress_hook:
             Callback where the caller can track progress of the operation
             as well as collect paths that failed to change Access Control.
-        :keyword str continuation:
+        :keyword str continuation_token:
             Optional continuation token that can be used to resume previously stopped operation.
         :keyword int batch_size:
             Optional. If data set size exceeds batch size then operation will be split into multiple
@@ -500,6 +502,8 @@ def update_access_control_recursive(self,
         :return: A summary of the recursive operations, including the count of successes and failures,
             as well as a continuation token in case the operation was terminated prematurely.
         :rtype: :class:`~azure.storage.filedatalake.AccessControlChangeResult`
+        :raises ~azure.core.exceptions.AzureError:
+            User can restart the operation using continuation_token field of AzureError if the token is available.
         """
         if not acl:
             raise ValueError("The Access Control List must be set for this operation")
@@ -526,7 +530,7 @@ def remove_access_control_recursive(self,
         :keyword func(~azure.storage.filedatalake.AccessControlChanges) progress_hook:
             Callback where the caller can track progress of the operation
             as well as collect paths that failed to change Access Control.
-        :keyword str continuation:
+        :keyword str continuation_token:
             Optional continuation token that can be used to resume previously stopped operation.
         :keyword int batch_size:
             Optional. If data set size exceeds batch size then operation will be split into multiple
@@ -548,6 +552,8 @@ def remove_access_control_recursive(self,
         :return: A summary of the recursive operations, including the count of successes and failures,
             as well as a continuation token in case the operation was terminated prematurely.
         :rtype: :class:`~azure.storage.filedatalake.AccessControlChangeResult`
+        :raises ~azure.core.exceptions.AzureError:
+            User can restart the operation using continuation_token field of AzureError if the token is available.
         """
         if not acl:
             raise ValueError("The Access Control List must be set for this operation")
@@ -613,8 +619,12 @@ def _set_access_control_internal(self, options, progress_hook, max_batches=None)
                 failure_count=total_failure_count),
                 continuation=last_continuation_token
                 if total_failure_count > 0 and not continue_on_failure else current_continuation_token)
+        except StorageErrorException as error:
+            error.continuation_token = last_continuation_token
+            process_storage_error(error)
         except AzureError as error:
-            raise DataLakeAclChangeFailedError(error, error.message, last_continuation_token)
+            error.continuation_token = last_continuation_token
+            raise error
 
     def _rename_path_options(self, rename_source, content_settings=None, metadata=None, **kwargs):
         # type: (Optional[ContentSettings], Optional[Dict[str, str]], **Any) -> Dict[str, Any]

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_version.py

@@ -4,4 +4,4 @@
 # license information.
 # --------------------------------------------------------------------------
 
-VERSION = "12.2.0b2"
+VERSION = "12.2.0"

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_path_client_async.py

@@ -9,7 +9,7 @@
 from .._shared.base_client_async import AsyncStorageAccountHostsMixin
 from .._path_client import PathClient as PathClientBase
 from .._models import DirectoryProperties, AccessControlChangeResult, AccessControlChangeFailure, \
-    AccessControlChangeCounters, AccessControlChanges, DataLakeAclChangeFailedError
+    AccessControlChangeCounters, AccessControlChanges
 from .._generated.aio import DataLakeStorageClient
 from ._data_lake_lease_async import DataLakeLeaseClient
 from .._generated.models import StorageErrorException
@@ -292,7 +292,7 @@ async def set_access_control_recursive(self,
         :keyword func(~azure.storage.filedatalake.AccessControlChanges) progress_hook:
             Callback where the caller can track progress of the operation
             as well as collect paths that failed to change Access Control.
-        :keyword str continuation:
+        :keyword str continuation_token:
             Optional continuation token that can be used to resume previously stopped operation.
         :keyword int batch_size:
             Optional. If data set size exceeds batch size then operation will be split into multiple
@@ -314,6 +314,8 @@ async def set_access_control_recursive(self,
         :return: A summary of the recursive operations, including the count of successes and failures,
             as well as a continuation token in case the operation was terminated prematurely.
         :rtype: :~azure.storage.filedatalake.AccessControlChangeResult`
+        :raises ~azure.core.exceptions.AzureError:
+            User can restart the operation using continuation_token field of AzureError if the token is available.
         """
         if not acl:
             raise ValueError("The Access Control List must be set for this operation")
@@ -339,7 +341,7 @@ async def update_access_control_recursive(self, acl, **kwargs):
         :keyword func(~azure.storage.filedatalake.AccessControlChanges) progress_hook:
             Callback where the caller can track progress of the operation
             as well as collect paths that failed to change Access Control.
-        :keyword str continuation:
+        :keyword str continuation_token:
             Optional continuation token that can be used to resume previously stopped operation.
         :keyword int batch_size:
             Optional. If data set size exceeds batch size then operation will be split into multiple
@@ -362,6 +364,8 @@ async def update_access_control_recursive(self, acl, **kwargs):
         :return: A summary of the recursive operations, including the count of successes and failures,
             as well as a continuation token in case the operation was terminated prematurely.
         :rtype: :~azure.storage.filedatalake.AccessControlChangeResult`
+        :raises ~azure.core.exceptions.AzureError:
+            User can restart the operation using continuation_token field of AzureError if the token is available.
         """
         if not acl:
             raise ValueError("The Access Control List must be set for this operation")
@@ -388,7 +392,7 @@ async def remove_access_control_recursive(self,
         :keyword func(~azure.storage.filedatalake.AccessControlChanges) progress_hook:
             Callback where the caller can track progress of the operation
             as well as collect paths that failed to change Access Control.
-        :keyword str continuation:
+        :keyword str continuation_token:
             Optional continuation token that can be used to resume previously stopped operation.
         :keyword int batch_size:
             Optional. If data set size exceeds batch size then operation will be split into multiple
@@ -410,6 +414,8 @@ async def remove_access_control_recursive(self,
         :return: A summary of the recursive operations, including the count of successes and failures,
             as well as a continuation token in case the operation was terminated prematurely.
         :rtype: :~azure.storage.filedatalake.AccessControlChangeResult`
+        :raises ~azure.core.exceptions.AzureError:
+            User can restart the operation using continuation_token field of AzureError if the token is available.
         """
         if not acl:
             raise ValueError("The Access Control List must be set for this operation")
@@ -475,8 +481,12 @@ async def _set_access_control_internal(self, options, progress_hook, max_batches
                 failure_count=total_failure_count),
                 continuation=last_continuation_token
                 if total_failure_count > 0 and not continue_on_failure else current_continuation_token)
+        except StorageErrorException as error:
+            error.continuation_token = last_continuation_token
+            process_storage_error(error)
         except AzureError as error:
-            raise DataLakeAclChangeFailedError(error, error.message, last_continuation_token)
+            error.continuation_token = last_continuation_token
+            raise error
 
     async def _rename_path(self, rename_source,
                            **kwargs):