[Storage][Test] Test multiple service versions (Azure#19039)

* adding infra for specifying service version from env var or by default

* changes for blob to show usage

* updates for storage-queue

* adding bens comment and assert policy for blob

* removing

* key change

* fixing way service version is read

* fixing an issue

* adding skipif the sv is too low

Author: seankane-msft

sdk/storage/azure-storage-blob/tests/_shared/service_versions.py

@@ -0,0 +1,42 @@
+from datetime import datetime
+from enum import Enum
+import os
+
+
+class ServiceVersion(str, Enum):
+
+    V2019_02_02 = "2019-02-02"
+    V2019_07_07 = "2019-07-07"
+    V2019_10_10 = "2019-10-10"
+    V2019_12_12 = "2019-12-12"
+    V2020_02_10 = "2020-02-10"
+    V2020_04_08 = "2020-04-08"
+    V2020_06_12 = "2020-06-12"
+    V2020_08_04 = "2020-08-04"
+
+
+service_version_map = {
+    "V2019_02_02": ServiceVersion.V2019_02_02,
+    "V2019_07_07": ServiceVersion.V2019_07_07,
+    "V2019_10_10": ServiceVersion.V2019_10_10,
+    "V2019_12_12": ServiceVersion.V2019_12_12,
+    "V2020_02_10": ServiceVersion.V2020_02_10,
+    "V2020_04_08": ServiceVersion.V2020_04_08,
+    "V2020_06_12": ServiceVersion.V2020_06_12,
+    "V2020_08_04": ServiceVersion.V2020_08_04,
+    "LATEST": ServiceVersion.V2020_08_04,
+    "LATEST_PLUS_1": ServiceVersion.V2020_06_12
+}
+
+
+def is_version_before(test_version):
+    """ Return True if the current version is after a given one or if the
+    service version is not set.
+    """
+    current_version = service_version_map.get(os.environ.get("AZURE_LIVE_TEST_SERVICE_VERSION"))
+    if not current_version:
+        return True
+    current_version_data = datetime.strptime(current_version, "%Y-%m-%d")
+    test_version_minimum = datetime.strptime(test_version, "%Y-%m-%d")
+    ret = current_version_data < test_version_minimum
+    return ret

sdk/storage/azure-storage-blob/tests/_shared/testcase.py

@@ -17,7 +17,7 @@
 import zlib
 import math
 import sys
-import string
+import os
 import random
 import re
 import logging
@@ -34,6 +34,7 @@
 except ImportError:
     from io import StringIO
 
+from azure.core.pipeline.policies import SansIOHTTPPolicy
 from azure.core.exceptions import ResourceNotFoundError, HttpResponseError
 from azure.core.credentials import AccessToken
 from azure.storage.blob import generate_account_sas, AccountSasPermissions, ResourceTypes
@@ -50,6 +51,8 @@
 except ImportError:
     from devtools_testutils import mgmt_settings_fake as settings
 
+from .service_versions import service_version_map
+
 import pytest
 
 
@@ -310,6 +313,32 @@ def generate_sas_token(self):
     def generate_fake_token(self):
         return FakeTokenCredential()
 
+    def _get_service_version(self, **kwargs):
+        env_version = service_version_map.get(os.environ.get("AZURE_LIVE_TEST_SERVICE_VERSION","LATEST"))
+        return kwargs.pop("service_version", env_version)
+
+    def create_storage_client(self, client, *args, **kwargs):
+        kwargs["api_version"] = self._get_service_version(**kwargs)
+        kwargs["_additional_pipeline_policies"] = [ApiVersionAssertPolicy(kwargs["api_version"])]
+        return client(*args, **kwargs)
+
+    def create_storage_client_from_conn_str(self, client, *args, **kwargs):
+        kwargs["api_version"] = self._get_service_version(**kwargs)
+        kwargs["_additional_pipeline_policies"] = [ApiVersionAssertPolicy(kwargs["api_version"])]
+        return client.from_connection_string(*args, **kwargs)
+
+
+class ApiVersionAssertPolicy(SansIOHTTPPolicy):
+    """
+    Assert the ApiVersion is set properly on the response
+    """
+
+    def __init__(self, api_version):
+        self.api_version = api_version
+
+    def on_request(self, request):
+        assert request.http_request.headers['x-ms-version'] == self.api_version
+
 
 def not_for_emulator(test):
     def skip_test_if_targeting_emulator(self):

sdk/storage/azure-storage-blob/tests/test_blob_tags.py

@@ -25,6 +25,8 @@
     BlobBlock, generate_account_sas, ResourceTypes, AccountSasPermissions, generate_container_sas,
     ContainerSasPermissions, BlobClient, generate_blob_sas, BlobSasPermissions)
 
+from _shared.service_versions import is_version_before, ServiceVersion
+
 #------------------------------------------------------------------------------
 
 TEST_CONTAINER_PREFIX = 'container'
@@ -34,7 +36,7 @@
 class StorageBlobTagsTest(StorageTestCase):
 
     def _setup(self, storage_account, key):
-        self.bsc = BlobServiceClient(self.account_url(storage_account, "blob"), credential=key)
+        self.bsc = self.create_storage_client(BlobServiceClient, self.account_url(storage_account, "blob"), credential=key)
         self.container_name = self.get_resource_name("container")
         if self.is_live:
             container = self.bsc.get_container_client(self.container_name)
@@ -90,6 +92,7 @@ def _create_container(self, prefix="container"):
 
     #-- test cases for blob tags ----------------------------------------------
 
+    @pytest.mark.skipif(is_version_before(ServiceVersion.V2019_12_12), reason="SV too low")
     @GlobalResourceGroupPreparer()
     @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
     def test_set_blob_tags(self, resource_group, location, storage_account, storage_account_key):
@@ -103,6 +106,7 @@ def test_set_blob_tags(self, resource_group, location, storage_account, storage_
         # Assert
         self.assertIsNotNone(resp)
 
+    @pytest.mark.skipif(is_version_before(ServiceVersion.V2019_12_12), reason="SV too low")
     @pytest.mark.playback_test_only
     @GlobalStorageAccountPreparer()
     def test_set_blob_tags_with_lease(self, resource_group, location, storage_account, storage_account_key):
@@ -126,6 +130,7 @@ def test_set_blob_tags_with_lease(self, resource_group, location, storage_accoun
 
         blob_client.delete_blob(lease=lease)
 
+    @pytest.mark.skipif(is_version_before(ServiceVersion.V2019_12_12), reason="SV too low")
     @pytest.mark.playback_test_only
     @GlobalStorageAccountPreparer()
     def test_set_blob_tags_for_a_version(self, resource_group, location, storage_account, storage_account_key):
@@ -142,6 +147,7 @@ def test_set_blob_tags_for_a_version(self, resource_group, location, storage_acc
         # Assert
         self.assertIsNotNone(resp)
 
+    @pytest.mark.skipif(is_version_before(ServiceVersion.V2019_12_12), reason="SV too low")
     @GlobalResourceGroupPreparer()
     @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
     def test_get_blob_tags(self, resource_group, location, storage_account, storage_account_key):
@@ -160,6 +166,7 @@ def test_get_blob_tags(self, resource_group, location, storage_account, storage_
         for key, value in resp.items():
             self.assertEqual(tags[key], value)
 
+    @pytest.mark.skipif(is_version_before(ServiceVersion.V2019_12_12), reason="SV too low")
     @GlobalResourceGroupPreparer()
     @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
     def test_get_blob_tags_for_a_snapshot(self, resource_group, location, storage_account, storage_account_key):
@@ -178,6 +185,7 @@ def test_get_blob_tags_for_a_snapshot(self, resource_group, location, storage_ac
         for key, value in resp.items():
             self.assertEqual(tags[key], value)
 
+    @pytest.mark.skipif(is_version_before(ServiceVersion.V2019_12_12), reason="SV too low")
     @GlobalResourceGroupPreparer()
     @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
     def test_upload_block_blob_with_tags(self, resource_group, location, storage_account, storage_account_key):
@@ -191,6 +199,7 @@ def test_upload_block_blob_with_tags(self, resource_group, location, storage_acc
         self.assertIsNotNone(resp)
         self.assertEqual(len(resp), 3)
 
+    @pytest.mark.skipif(is_version_before(ServiceVersion.V2019_12_12), reason="SV too low")
     @GlobalResourceGroupPreparer()
     @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
     def test_get_blob_properties_returns_tags_num(self, resource_group, location, storage_account, storage_account_key):
@@ -206,6 +215,7 @@ def test_get_blob_properties_returns_tags_num(self, resource_group, location, st
         self.assertEqual(resp.tag_count, len(tags))
         self.assertEqual(downloaded.properties.tag_count, len(tags))
 
+    @pytest.mark.skipif(is_version_before(ServiceVersion.V2019_12_12), reason="SV too low")
     @GlobalResourceGroupPreparer()
     @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
     def test_create_append_blob_with_tags(self, resource_group, location, storage_account, storage_account_key):
@@ -219,6 +229,7 @@ def test_create_append_blob_with_tags(self, resource_group, location, storage_ac
         self.assertIsNotNone(resp)
         self.assertEqual(len(resp), 3)
 
+    @pytest.mark.skipif(is_version_before(ServiceVersion.V2019_12_12), reason="SV too low")
     @GlobalResourceGroupPreparer()
     @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
     def test_create_page_blob_with_tags(self, resource_group, location, storage_account, storage_account_key):
@@ -232,6 +243,7 @@ def test_create_page_blob_with_tags(self, resource_group, location, storage_acco
         self.assertIsNotNone(resp)
         self.assertEqual(len(resp), 3)
 
+    @pytest.mark.skipif(is_version_before(ServiceVersion.V2019_12_12), reason="SV too low")
     @GlobalResourceGroupPreparer()
     @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
     def test_commit_block_list_with_tags(self, resource_group, location, storage_account, storage_account_key):
@@ -255,6 +267,7 @@ def test_commit_block_list_with_tags(self, resource_group, location, storage_acc
         self.assertIsNotNone(resp)
         self.assertEqual(len(resp), len(tags))
 
+    @pytest.mark.skipif(is_version_before(ServiceVersion.V2019_12_12), reason="SV too low")
     @GlobalResourceGroupPreparer()
     @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
     def test_start_copy_from_url_with_tags(self, resource_group, location, storage_account, storage_account_key):
@@ -284,6 +297,7 @@ def test_start_copy_from_url_with_tags(self, resource_group, location, storage_a
         self.assertIsNotNone(resp)
         self.assertEqual(len(resp), len(tags))
 
+    @pytest.mark.skipif(is_version_before(ServiceVersion.V2019_12_12), reason="SV too low")
     @GlobalResourceGroupPreparer()
     @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
     def test_list_blobs_returns_tags(self, resource_group, location, storage_account, storage_account_key):
@@ -299,6 +313,7 @@ def test_list_blobs_returns_tags(self, resource_group, location, storage_account
             for key, value in blob.tags.items():
                 self.assertEqual(tags[key], value)
 
+    @pytest.mark.skipif(is_version_before(ServiceVersion.V2019_12_12), reason="SV too low")
     @pytest.mark.playback_test_only
     @GlobalStorageAccountPreparer()
     def test_filter_blobs(self, resource_group, location, storage_account, storage_account_key):
@@ -328,6 +343,7 @@ def test_filter_blobs(self, resource_group, location, storage_account, storage_a
         self.assertEqual(items_on_page2[0]['tags']['tag1'], 'firsttag')
         self.assertEqual(items_on_page2[0]['tags']['tag2'], 'secondtag')
 
+    @pytest.mark.skipif(is_version_before(ServiceVersion.V2019_12_12), reason="SV too low")
     @pytest.mark.live_test_only
     @GlobalResourceGroupPreparer()
     @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
@@ -360,6 +376,7 @@ def test_filter_blobs_using_account_sas(self, resource_group, location, storage_
         items_on_page1 = list(first_page)
         self.assertEqual(1, len(items_on_page1))
 
+    @pytest.mark.skipif(is_version_before(ServiceVersion.V2019_12_12), reason="SV too low")
     @pytest.mark.live_test_only
     @GlobalResourceGroupPreparer()
     @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')

sdk/storage/azure-storage-queue/tests/_shared/service_versions.py

@@ -0,0 +1,27 @@
+from enum import Enum
+
+
+class ServiceVersion(str, Enum):
+
+    V2019_02_02 = "2019-02-02"
+    V2019_07_07 = "2019-07-07"
+    V2019_10_10 = "2019-10-10"
+    V2019_12_12 = "2019-12-12"
+    V2020_02_10 = "2020-02-10"
+    V2020_04_08 = "2020-04-08"
+    V2020_06_12 = "2020-06-12"
+    V2020_08_04 = "2020-08-04"
+
+
+service_version_map = {
+    "V2019_02_02": ServiceVersion.V2019_02_02,
+    "V2019_07_07": ServiceVersion.V2019_07_07,
+    "V2019_10_10": ServiceVersion.V2019_10_10,
+    "V2019_12_12": ServiceVersion.V2019_12_12,
+    "V2020_02_10": ServiceVersion.V2020_02_10,
+    "V2020_04_08": ServiceVersion.V2020_04_08,
+    "V2020_06_12": ServiceVersion.V2020_06_12,
+    "V2020_08_04": ServiceVersion.V2020_08_04,
+    "LATEST": ServiceVersion.V2020_08_04,
+    "LATEST_PLUS_1": ServiceVersion.V2020_06_12
+}

sdk/storage/azure-storage-queue/tests/_shared/testcase.py

@@ -34,6 +34,7 @@
 except ImportError:
     from io import StringIO
 
+from azure.core.pipeline.policies import SansIOHTTPPolicy
 from azure.core.exceptions import ResourceNotFoundError, HttpResponseError
 from azure.core.credentials import AccessToken
 from azure.storage.queue import generate_account_sas, AccountSasPermissions, ResourceTypes
@@ -52,6 +53,8 @@
 
 import pytest
 
+from .service_versions import service_version_map
+
 
 LOGGING_FORMAT = '%(asctime)s %(name)-20s %(levelname)-5s %(message)s'
 os.environ['AZURE_STORAGE_ACCOUNT_NAME'] = STORAGE_ACCOUNT_NAME
@@ -310,13 +313,39 @@ def generate_sas_token(self):
     def generate_fake_token(self):
         return FakeTokenCredential()
 
+    def _get_service_version(self, **kwargs):
+        env_version = service_version_map.get(os.environ.get("AZURE_LIVE_TEST_SERVICE_VERSION","LATEST"))
+        return kwargs.pop("service_version", env_version)
+
+    def create_storage_client(self, client, *args, **kwargs):
+        kwargs["api_version"] = self._get_service_version(**kwargs)
+        kwargs["_additional_pipeline_policies"] = [ApiVersionAssertPolicy(kwargs["api_version"])]
+        return client(*args, **kwargs)
+
+    def create_storage_client_from_conn_str(self, client, *args, **kwargs):
+        kwargs["api_version"] = self._get_service_version(**kwargs)
+        kwargs["_additional_pipeline_policies"] = [ApiVersionAssertPolicy(kwargs["api_version"])]
+        return client.from_connection_string(*args, **kwargs)
+
 
 def not_for_emulator(test):
     def skip_test_if_targeting_emulator(self):
         test(self)
     return skip_test_if_targeting_emulator
 
 
+class ApiVersionAssertPolicy(SansIOHTTPPolicy):
+    """
+    Assert the ApiVersion is set properly on the response
+    """
+
+    def __init__(self, api_version):
+        self.api_version = api_version
+
+    def on_request(self, request):
+        assert request.http_request.headers['x-ms-version'] == self.api_version
+
+
 class RetryCounter(object):
     def __init__(self):
         self.count = 0

sdk/storage/azure-storage-queue/tests/test_queue_service_stats.py

@@ -49,7 +49,7 @@ def override_response_body_with_live_status(response):
     @StorageAccountPreparer(name_prefix='pyacrstorage', sku='Standard_RAGRS', random_name_enabled=True)
     def test_queue_service_stats_f(self, resource_group, location, storage_account, storage_account_key):
         # Arrange
-        qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key)
+        qsc = self.create_storage_client(QueueServiceClient, self.account_url(storage_account, "queue"), storage_account_key)
 
         # Act
         stats = qsc.get_service_stats(raw_response_hook=self.override_response_body_with_live_status)
@@ -60,11 +60,10 @@ def test_queue_service_stats_f(self, resource_group, location, storage_account,
     @StorageAccountPreparer(name_prefix='pyacrstorage', sku='Standard_RAGRS', random_name_enabled=True)
     def test_queue_service_stats_when_unavailable(self, resource_group, location, storage_account, storage_account_key):
         # Arrange
-        qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key)
+        qsc = self.create_storage_client(QueueServiceClient, self.account_url(storage_account, "queue"), storage_account_key)
 
         # Act
-        stats = qsc.get_service_stats(
-            raw_response_hook=self.override_response_body_with_unavailable_status)
+        stats = qsc.get_service_stats(raw_response_hook=self.override_response_body_with_unavailable_status)
 
         # Assert
         self._assert_stats_unavailable(stats)

sdk/storage/platform-matrix-all-versions.json

@@ -0,0 +1,26 @@
+{
+    "displayNames": {
+      "--disablecov": "",
+      "false": "",
+      "true": ""
+    },
+    "matrix": {
+        "Agent": {
+          "ubuntu-18.04": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" },
+          "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" },
+          "macOS-10.15": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" }
+        },
+        "PythonVersion": [ "pypy3", "2.7", "3.6", "3.7", "3.8", "3.9" ],
+        "CoverageArg": "--disablecov",
+        "TestSamples": "false",
+        "AZURE_LIVE_TEST_SERVICE_VERSION": [
+            "V2019_02_02",
+            "V2019_07_07",
+            "V2019_12_12",
+            "V2020_02_10",
+            "V2020_04_08",
+            "V2020_06_12",
+            "V2020_08_04"
+        ]
+    }
+}

sdk/storage/tests.yml

@@ -26,6 +26,12 @@ stages:
         MatrixReplace:
           # Use dedicated storage pool in canadacentral with higher memory capacity
           - Pool=(.*)-general/$1-storage
+        ${{ if contains(variables['Build.DefinitionName'], 'tests-weekly') }}:
+          MatrixConfigs:
+            - Name: Storage_all_versions_live_test
+              Path: sdk/storage/platform-matrix-all-versions.json
+              Selection: sparse
+              GenerateVMJobs: true
         EnvVars:
           STORAGE_ACCOUNT_NAME: $(python-storage-storage-account-name)
           STORAGE_ACCOUNT_KEY: $(python-storage-storage-account-key)
@@ -51,4 +57,4 @@ stages:
           AZURE_TENANT_ID: $(aad-azure-sdk-test-tenant-id)
           AZURE_SUBSCRIPTION_ID: $(azure-subscription-id)
           AZURE_CLIENT_SECRET: $(aad-azure-sdk-test-client-secret)
-          AZURE_CLIENT_ID: $(aad-azure-sdk-test-client-id)
+          AZURE_CLIENT_ID: $(aad-azure-sdk-test-client-id)