Enabled flexible local testing and ensure consistency among services (Azure#18907)

* Enabled flexible local testing and ensured consistency among services

* fixed imports

Author: tasherif-msft

.gitignore

@@ -92,10 +92,17 @@ venv/
 code_reports
 
 # Azure Storage test credentials
+sdk/storage/azure-storage-blob/tests/_shared/settings_real.py
+sdk/storage/azure-storage-queue/tests/_shared/settings_real.py
+sdk/storage/azure-storage-file-share/tests/_shared/settings_real.py
+sdk/storage/azure-storage-file-datalake/tests/settings_real.py
+
+# The locations below are deprecated - keep to prevent any accidental secrets leakage ==========
 sdk/storage/azure-storage-blob/tests/settings_real.py
 sdk/storage/azure-storage-queue/tests/settings_real.py
 sdk/storage/azure-storage-file-share/tests/settings_real.py
-sdk/storage/azure-storage-file-datalake/tests/settings_real.py
+# ==============================================================================================
+
 *.code-workspace
 sdk/cosmos/azure-cosmos/test/test_config.py
 

sdk/storage/azure-storage-blob/tests/_shared/testcase.py

@@ -5,10 +5,6 @@
 # license information.
 # --------------------------------------------------------------------------
 from __future__ import division
-from contextlib import contextmanager
-import copy
-import inspect
-import os
 import os.path
 import time
 from datetime import datetime, timedelta
@@ -43,8 +39,10 @@
 from azure.storage.blob import generate_account_sas, AccountSasPermissions, ResourceTypes
 from azure.mgmt.storage.models import StorageAccount, Endpoints
 try:
+    # Running locally - use configuration in settings_real.py
     from .settings_real import *
 except ImportError:
+    # Running on the pipeline - use fake values in order to create rg, etc.
     from .settings_fake import *
 
 try:

sdk/storage/azure-storage-blob/tests/blob_performance.py

@@ -12,7 +12,7 @@
     ContainerClient,
     BlobClient,
 )
-import tests.settings_real as settings
+import _shared.settings_real as settings
 
 # Warning:
 # This script will take a while to run with everything enabled.

sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_set_blob_properties_with_if_none_match_fail.yaml

@@ -3,19 +3,19 @@ interactions:
     body: null
     headers:
       Accept:
-      - '*/*'
+      - application/xml
       Accept-Encoding:
       - gzip, deflate
       Connection:
       - keep-alive
       Content-Length:
       - '0'
       User-Agent:
-      - azsdk-python-storage-blob/12.0.0b5 Python/3.6.3 (Windows-10-10.0.18362-SP0)
+      - azsdk-python-storage-blob/12.9.0b1 Python/3.8.5 (Windows-10-10.0.19041-SP0)
       x-ms-date:
-      - Fri, 25 Oct 2019 17:22:42 GMT
+      - Tue, 25 May 2021 05:09:40 GMT
       x-ms-version:
-      - '2019-02-02'
+      - '2020-08-04'
     method: PUT
     uri: https://storagename.blob.core.windows.net/utcontainerb1921f2b?restype=container
   response:
@@ -25,23 +25,23 @@ interactions:
       content-length:
       - '0'
       date:
-      - Fri, 25 Oct 2019 17:22:41 GMT
+      - Tue, 25 May 2021 05:09:41 GMT
       etag:
-      - '"0x8D7596FF19BD234"'
+      - '"0x8D91F3B4D9376CA"'
       last-modified:
-      - Fri, 25 Oct 2019 17:22:42 GMT
+      - Tue, 25 May 2021 05:09:41 GMT
       server:
       - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
       x-ms-version:
-      - '2019-02-02'
+      - '2020-08-04'
     status:
       code: 201
       message: Created
 - request:
     body: hello world
     headers:
       Accept:
-      - '*/*'
+      - application/xml
       Accept-Encoding:
       - gzip, deflate
       Connection:
@@ -53,13 +53,13 @@ interactions:
       If-None-Match:
       - '*'
       User-Agent:
-      - azsdk-python-storage-blob/12.0.0b5 Python/3.6.3 (Windows-10-10.0.18362-SP0)
+      - azsdk-python-storage-blob/12.9.0b1 Python/3.8.5 (Windows-10-10.0.19041-SP0)
       x-ms-blob-type:
       - BlockBlob
       x-ms-date:
-      - Fri, 25 Oct 2019 17:22:42 GMT
+      - Tue, 25 May 2021 05:09:40 GMT
       x-ms-version:
-      - '2019-02-02'
+      - '2020-08-04'
     method: PUT
     uri: https://storagename.blob.core.windows.net/utcontainerb1921f2b/blob1
   response:
@@ -71,37 +71,39 @@ interactions:
       content-md5:
       - XrY7u+Ae7tCTyyK7j1rNww==
       date:
-      - Fri, 25 Oct 2019 17:22:41 GMT
+      - Tue, 25 May 2021 05:09:41 GMT
       etag:
-      - '"0x8D7596FF1A4D6EC"'
+      - '"0x8D91F3B4DB76284"'
       last-modified:
-      - Fri, 25 Oct 2019 17:22:42 GMT
+      - Tue, 25 May 2021 05:09:41 GMT
       server:
       - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
       x-ms-content-crc64:
       - vo7q9sPVKY0=
       x-ms-request-server-encrypted:
       - 'true'
       x-ms-version:
-      - '2019-02-02'
+      - '2020-08-04'
+      x-ms-version-id:
+      - '2021-05-25T05:09:41.7031300Z'
     status:
       code: 201
       message: Created
 - request:
     body: null
     headers:
       Accept:
-      - '*/*'
+      - application/xml
       Accept-Encoding:
       - gzip, deflate
       Connection:
       - keep-alive
       User-Agent:
-      - azsdk-python-storage-blob/12.0.0b5 Python/3.6.3 (Windows-10-10.0.18362-SP0)
+      - azsdk-python-storage-blob/12.9.0b1 Python/3.8.5 (Windows-10-10.0.19041-SP0)
       x-ms-date:
-      - Fri, 25 Oct 2019 17:22:42 GMT
+      - Tue, 25 May 2021 05:09:41 GMT
       x-ms-version:
-      - '2019-02-02'
+      - '2020-08-04'
     method: HEAD
     uri: https://storagename.blob.core.windows.net/utcontainerb1921f2b/blob1
   response:
@@ -117,11 +119,11 @@ interactions:
       content-type:
       - application/octet-stream
       date:
-      - Fri, 25 Oct 2019 17:22:41 GMT
+      - Tue, 25 May 2021 05:09:41 GMT
       etag:
-      - '"0x8D7596FF1A4D6EC"'
+      - '"0x8D91F3B4DB76284"'
       last-modified:
-      - Fri, 25 Oct 2019 17:22:42 GMT
+      - Tue, 25 May 2021 05:09:41 GMT
       server:
       - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
       x-ms-access-tier:
@@ -131,60 +133,64 @@ interactions:
       x-ms-blob-type:
       - BlockBlob
       x-ms-creation-time:
-      - Fri, 25 Oct 2019 17:22:42 GMT
+      - Tue, 25 May 2021 05:09:41 GMT
+      x-ms-is-current-version:
+      - 'true'
       x-ms-lease-state:
       - available
       x-ms-lease-status:
       - unlocked
       x-ms-server-encrypted:
       - 'true'
       x-ms-version:
-      - '2019-02-02'
+      - '2020-08-04'
+      x-ms-version-id:
+      - '2021-05-25T05:09:41.7031300Z'
     status:
       code: 200
       message: OK
 - request:
     body: null
     headers:
       Accept:
-      - '*/*'
+      - application/xml
       Accept-Encoding:
       - gzip, deflate
       Connection:
       - keep-alive
       Content-Length:
       - '0'
       If-None-Match:
-      - '"0x8D7596FF1A4D6EC"'
+      - '"0x8D91F3B4DB76284"'
       User-Agent:
-      - azsdk-python-storage-blob/12.0.0b5 Python/3.6.3 (Windows-10-10.0.18362-SP0)
+      - azsdk-python-storage-blob/12.9.0b1 Python/3.8.5 (Windows-10-10.0.19041-SP0)
       x-ms-blob-content-disposition:
       - inline
       x-ms-blob-content-language:
       - spanish
       x-ms-date:
-      - Fri, 25 Oct 2019 17:22:42 GMT
+      - Tue, 25 May 2021 05:09:41 GMT
       x-ms-version:
-      - '2019-02-02'
+      - '2020-08-04'
     method: PUT
     uri: https://storagename.blob.core.windows.net/utcontainerb1921f2b/blob1?comp=properties
   response:
     body:
       string: "\uFEFF<?xml version=\"1.0\" encoding=\"utf-8\"?><Error><Code>ConditionNotMet</Code><Message>The
-        condition specified using HTTP conditional header(s) is not met.\nRequestId:363a2d2e-901e-00d9-6c58-8b2793000000\nTime:2019-10-25T17:22:42.7538498Z</Message></Error>"
+        condition specified using HTTP conditional header(s) is not met.\nRequestId:553c318a-a01e-0052-0f24-513148000000\nTime:2021-05-25T05:09:42.0713805Z</Message></Error>"
     headers:
       content-length:
       - '252'
       content-type:
       - application/xml
       date:
-      - Fri, 25 Oct 2019 17:22:41 GMT
+      - Tue, 25 May 2021 05:09:41 GMT
       server:
       - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
       x-ms-error-code:
       - ConditionNotMet
       x-ms-version:
-      - '2019-02-02'
+      - '2020-08-04'
     status:
       code: 412
       message: The condition specified using HTTP conditional header(s) is not met.

sdk/storage/azure-storage-file-share/tests/_shared/settings_fake.py

@@ -0,0 +1,17 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+STORAGE_ACCOUNT_NAME = ""
+STORAGE_ACCOUNT_KEY = ""
+
+ACCOUNT_URL_SUFFIX = 'core.windows.net'
+RUN_IN_LIVE = "False"
+SKIP_LIVE_RECORDING = "True"
+
+PROTOCOL = "https"
+
+ENABLE_LOGGING = True
+