Run MHSM tests weekly, disable attestation in Canary (Azure#36651)

* Run MHSM tests weekly, disable attestation in Canary

* Fixed cspell issues.

Author: vcolin7

.vscode/cspell.json

@@ -152,10 +152,11 @@
     "sdk/identity/azure-identity/**",
     "sdk/keyvault/azure-security-keyvault-administration/**",
     "sdk/keyvault/azure-security-keyvault-certificates/**",
-    "sdk/keyvault/azure-security-test-keyvault-jca/**",
     "sdk/keyvault/azure-security-keyvault-jca/**",
-    "sdk/keyvault/azure-security-keyvault-secrets/**",
     "sdk/keyvault/azure-security-keyvault-keys/**",
+    "sdk/keyvault/azure-security-keyvault-secrets/**",
+    "sdk/keyvault/azure-security-test-keyvault-jca/**",
+    "sdk/keyvault/test-resources.json",
     "sdk/formrecognizer/azure-ai-formrecognizer/**",
     "sdk/core/azure-core/**",
     "sdk/maps/azure-maps-render/**",

sdk/keyvault/azure-security-keyvault-keys/src/test/java/com/azure/security/keyvault/keys/KeyAsyncClientTest.java

@@ -587,7 +587,7 @@ public void listKeys(HttpClient httpClient, KeyServiceVersion serviceVersion) {
     @MethodSource("getTestParameters")
     public void releaseKey(HttpClient httpClient, KeyServiceVersion serviceVersion) {
         // TODO: Remove assumption once Key Vault allows for creating exportable keys.
-        Assumptions.assumeTrue(runManagedHsmTest);
+        Assumptions.assumeTrue(runManagedHsmTest && runReleaseKeyTest);
 
         createKeyAsyncClient(httpClient, serviceVersion);
 

sdk/keyvault/azure-security-keyvault-keys/src/test/java/com/azure/security/keyvault/keys/KeyClientTest.java

@@ -566,7 +566,7 @@ public void listKeyVersions(HttpClient httpClient, KeyServiceVersion serviceVers
     @MethodSource("getTestParameters")
     public void releaseKey(HttpClient httpClient, KeyServiceVersion serviceVersion) {
         // TODO: Remove assumption once Key Vault allows for creating exportable keys.
-        Assumptions.assumeTrue(runManagedHsmTest);
+        Assumptions.assumeTrue(runManagedHsmTest && runReleaseKeyTest);
 
         createKeyClient(httpClient, serviceVersion);
 

sdk/keyvault/azure-security-keyvault-keys/src/test/java/com/azure/security/keyvault/keys/KeyClientTestBase.java

@@ -21,6 +21,7 @@
 import com.azure.core.http.policy.RetryStrategy;
 import com.azure.core.http.policy.UserAgentPolicy;
 import com.azure.core.http.rest.Response;
+import com.azure.core.test.TestMode;
 import com.azure.core.test.TestProxyTestBase;
 import com.azure.core.test.models.BodilessMatcher;
 import com.azure.core.test.models.CustomMatcher;
@@ -79,6 +80,8 @@ public abstract class KeyClientTestBase extends TestProxyTestBase {
 
     protected boolean isHsmEnabled = false;
     protected boolean runManagedHsmTest = false;
+    protected boolean runReleaseKeyTest = getTestMode() == TestMode.PLAYBACK
+        || Configuration.getGlobalConfiguration().get("AZURE_KEYVAULT_ATTESTATION_URL") != null;
 
     @Override
     protected String getTestName() {

sdk/keyvault/platform-matrix.json

@@ -1,9 +1,12 @@
 {
   "displayNames": {
+    "@{ enableAttestation = $true }": "",
+    "@{ enableAttestation = $false }": "NoAttestation",
     "@{ enableHsm = $true }": "HSM"
   },
   "matrix": {
-    "$IMPORT": "eng/pipelines/templates/stages/platform-matrix.json"
+    "$IMPORT": "eng/pipelines/templates/stages/platform-matrix.json",
+    "ArmTemplateParameters": "@{ enableAttestation = $true }"
   },
   "exclude": [
     {

sdk/keyvault/test-resources.json

@@ -69,6 +69,13 @@
         "description": "The location of the Managed HSM. By default, this is 'northcentralus'."
       }
     },
+    "enableAttestation": {
+      "type": "bool",
+      "defaultValue": true,
+      "metadata": {
+        "description": "Whether to enable deployment of attestation resources. The default is true."
+      }
+    },
     "enableHsm": {
       "type": "bool",
       "defaultValue": false,
@@ -405,6 +412,7 @@
       "type": "Microsoft.Web/serverfarms",
       "apiVersion": "2020-12-01",
       "name": "[variables('attestationFarm')]",
+      "condition": "[parameters('enableAttestation')]",
       "location": "[parameters('location')]",
       "kind": "linux",
       "sku": {
@@ -418,6 +426,7 @@
       "type": "Microsoft.Web/sites",
       "apiVersion": "2020-12-01",
       "name": "[variables('attestationSite')]",
+      "condition": "[parameters('enableAttestation')]",
       "dependsOn": [
         "[resourceId('Microsoft.Web/serverfarms', variables('attestationFarm'))]"
       ],
@@ -556,6 +565,7 @@
     },
     "AZURE_KEYVAULT_ATTESTATION_URL": {
       "type": "string",
+      "condition": "[parameters('enableAttestation')]",
       "value": "[format('https://{0}/', reference(variables('attestationSite')).defaultHostName)]"
     },
     "KEY_VAULT_ENDPOINT_SUFFIX": {

sdk/keyvault/tests.yml

@@ -11,6 +11,9 @@ stages:
       CloudConfig:
         Public:
           SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
+          ${{ if not(contains(variables['Build.DefinitionName'], 'tests-weekly')) }}:
+            MatrixFilters:
+              - ArmTemplateParameters=^(?!.*enableHsm.*true)
         UsGov:
           SubscriptionConfiguration: $(sub-config-gov-test-resources)
           MatrixFilters:
@@ -26,6 +29,9 @@ stages:
           # Given test coverage of non-canary regions we probably don't need to test in canary.
           MatrixFilters:
             - ArmTemplateParameters=^(?!.*enableHsm.*true)
+          # Some resource providers required for attestation are not supported in canary.
+          MatrixReplace:
+            - 'ArmTemplateParameters=(.*)enableAttestation.*?\$true(.*)/$1enableAttestation \= $false$2'
       MatrixConfigs:
         - Name: Key_Vault_live_test
           Path: sdk/keyvault/platform-matrix.json