Re enable batch endpoints/deployments e2etests (Azure#27177)

* Re enable batch/online endpoints e2etests

Author: nancy-mejia

sdk/ml/azure-ai-ml/tests/batch_services/e2etests/test_batch_deployment.py

@@ -35,9 +35,7 @@ def deployEndpointAndDeployment(client: MLClient, endpoint: BatchEndpoint, deplo
     client.batch_endpoints.begin_delete(name=endpoint.name)
 
 
-@pytest.mark.skip(
-    reason="Tests failing in internal automation due to lack of quota. Cannot record or run in live mode."
-)
+@pytest.mark.e2etest
 @pytest.mark.usefixtures("recorded_test")
 @pytest.mark.production_experience_test
 class TestBatchDeployment(AzureRecordedTestCase):
@@ -76,14 +74,13 @@ def test_batch_deployment(self, client: MLClient, data_with_2_versions: str) ->
         )
         client.batch_endpoints.begin_delete(name=endpoint.name)
 
-    @pytest.mark.e2etest
-    def test_batch_deployment_dependency_label_resolution(self, client: MLClient, randstr: Callable[[], str]) -> None:
+    def test_batch_deployment_dependency_label_resolution(self, client: MLClient, randstr: Callable[[], str], rand_batch_name: Callable[[], str], rand_batch_deployment_name: Callable[[], str]) -> None:
         endpoint_yaml = "./tests/test_configs/endpoints/batch/batch_endpoint_mlflow_new.yaml"
-        name = "batch-ept-" + uuid.uuid4().hex[:15]
+        name = rand_batch_name("name")
         deployment_yaml = "./tests/test_configs/deployments/batch/batch_deployment_mlflow_new.yaml"
-        deployment_name = "batch-dpm-" + uuid.uuid4().hex[:15]
+        deployment_name = rand_batch_deployment_name("deployment_name")
 
-        environment_name = randstr()
+        environment_name = randstr("environment_name")
         environment_versions = ["foo", "bar"]
 
         for version in environment_versions:
@@ -94,7 +91,7 @@ def test_batch_deployment_dependency_label_resolution(self, client: MLClient, ra
                 )
             )
 
-        model_name = randstr()
+        model_name = randstr("model_name")
         model_versions = ["1", "2"]
 
         for version in model_versions:
@@ -134,15 +131,16 @@ def test_batch_deployment_dependency_label_resolution(self, client: MLClient, ra
         )
         assert resolved_model.asset_name == model_name and resolved_model.asset_version == model_versions[-1]
 
-    @pytest.mark.e2etest
-    def test_batch_job_download(self, client: MLClient, tmp_path: Path) -> str:
+    def test_batch_job_download(self, client: MLClient, tmp_path: Path, rand_batch_name: Callable[[], str], rand_batch_deployment_name: Callable[[], str]) -> str:
+        endpoint_name = rand_batch_name("name")
         endpoint = load_batch_endpoint(
             "./tests/test_configs/endpoints/batch/batch_endpoint_mlflow_new.yaml",
-            params_override=[{"name": "batch-ept-" + uuid.uuid4().hex[:15]}],
+            params_override=[{"name": endpoint_name}],
         )
+        deployment_name = rand_batch_deployment_name("deployment_name")
         deployment = load_batch_deployment(
             "./tests/test_configs/deployments/batch/batch_deployment_quick.yaml",
-            params_override=[{"endpoint_name": endpoint.name}, {"name": "batch-dpm-" + uuid.uuid4().hex[:15]}],
+            params_override=[{"endpoint_name": endpoint.name}, {"name": deployment_name}],
         )
         endpoint.traffic = {deployment.name: 100}
 

sdk/ml/azure-ai-ml/tests/batch_services/e2etests/test_batch_endpoint.py

@@ -1,32 +1,29 @@
-import uuid
+from typing import Callable
 
 import pytest
 
 from azure.ai.ml import MLClient, load_batch_endpoint, load_batch_deployment
-from azure.ai.ml.entities._assets import Model
 from azure.ai.ml.entities._inputs_outputs import Input
 from azure.core.exceptions import ResourceNotFoundError
 
 from devtools_testutils import AzureRecordedTestCase
 
 
 @pytest.mark.e2etest
+@pytest.mark.usefixtures("recorded_test")
 @pytest.mark.production_experience_test
 class TestBatchEndpoint(AzureRecordedTestCase):
-    @pytest.mark.skip(
-        reason="Tests failing in internal automation due to lack of quota. Cannot record or run in live mode."
-    )
-    def test_batch_endpoint_create_and_invoke(
-        self, client: MLClient, data_with_2_versions: str, batch_endpoint_model: Model
+    def test_batch_endpoint_create(
+        self, client: MLClient, rand_batch_name: Callable[[], str]
     ) -> None:
         endpoint_yaml = "./tests/test_configs/endpoints/batch/batch_endpoint.yaml"
-        name = "be-e2e-" + uuid.uuid4().hex[:25]
+        name = rand_batch_name("name")
         # Bug in MFE that batch endpoint properties are not preserved, uncomment below after it's fixed in MFE
         # properties = {"property1": "value1", "property2": "value2"}
         endpoint = load_batch_endpoint(endpoint_yaml)
         endpoint.name = name
         # endpoint.properties = properties
-        obj = client.batch_endpoints.begin_create_or_update(endpoint=endpoint, no_wait=False)
+        obj = client.batch_endpoints.begin_create_or_update(endpoint=endpoint)
         obj = obj.result()
         assert obj is not None
         assert name == obj.name
@@ -45,16 +42,13 @@ def test_batch_endpoint_create_and_invoke(
 
         raise Exception(f"Batch endpoint {name} is supposed to be deleted.")
 
-    @pytest.mark.skip(
-        reason="Tests failing in internal automation due to lack of quota. Cannot record or run in live mode."
-    )
-    @pytest.mark.e2etest
+
     @pytest.mark.usefixtures("light_gbm_model")
-    def test_mlflow_batch_endpoint_create_and_update(self, client: MLClient) -> None:
+    def test_mlflow_batch_endpoint_create_and_update(self, client: MLClient, rand_batch_name: Callable[[], str]) -> None:
         # light_gbm_model fixture is not used directly, but it makes sure the model being used by the batch endpoint exists
 
         endpoint_yaml = "./tests/test_configs/endpoints/batch/batch_endpoint_mlflow.yaml"
-        name = "be-e2e-" + uuid.uuid4().hex[:25]
+        name = rand_batch_name("name")
         endpoint = load_batch_endpoint(endpoint_yaml)
         endpoint.name = name
         obj = client.batch_endpoints.begin_create_or_update(endpoint=endpoint)
@@ -76,14 +70,14 @@ def test_mlflow_batch_endpoint_create_and_update(self, client: MLClient) -> None
         raise Exception(f"Batch endpoint {name} is supposed to be deleted.")
 
 
-    def test_batch_invoke(self, client: MLClient, variable_recorder) -> None:
+    def test_batch_invoke(self, client: MLClient, rand_batch_name: Callable[[], str], rand_batch_deployment_name: Callable[[], str]) -> None:
         endpoint_yaml = "./tests/test_configs/endpoints/batch/simple_batch_endpoint.yaml"
-        endpoint_name = variable_recorder.get_or_record("endpoint_name", "be-e2e-" + uuid.uuid4().hex[:15])
+        endpoint_name = rand_batch_name("endpoint_name")
         endpoint = load_batch_endpoint(endpoint_yaml)
         endpoint.name = endpoint_name
 
         deployment_yaml = "./tests/test_configs/deployments/batch/batch_deployment_3.yaml"
-        deployment_name = variable_recorder.get_or_record("deployment_name", "batch-dpm-" + uuid.uuid4().hex[:15])
+        deployment_name = rand_batch_deployment_name("deployment_name")
 
         deployment = load_batch_deployment(deployment_yaml)
         deployment.endpoint_name = endpoint_name

sdk/ml/azure-ai-ml/tests/conftest.py

@@ -71,6 +71,7 @@ def add_sanitizers(test_proxy, fake_datastore_key):
     add_body_key_sanitizer(json_path="$.properties.properties.hash_sha256", value="0000000000000")
     add_body_key_sanitizer(json_path="$.properties.properties.hash_version", value="0000000000000")
     add_body_key_sanitizer(json_path="$.properties.properties.['azureml.git.dirty']", value="fake_git_dirty_value")
+    add_body_key_sanitizer(json_path="$.accessToken", value="Sanitized")
     add_general_regex_sanitizer(value="", regex=f"\\u0026tid={os.environ.get('ML_TENANT_ID')}")
     add_general_string_sanitizer(value="", target=f"&tid={os.environ.get('ML_TENANT_ID')}")
     add_general_regex_sanitizer(
@@ -205,6 +206,45 @@ def generate_random_string(variable_name: str):
 
     return generate_random_string
 
+@pytest.fixture
+def rand_batch_name(variable_recorder: VariableRecorder) -> Callable[[str], str]:
+    """return a random batch endpoint name string  e.g. batch-ept-xxx"""
+
+    def generate_random_string(variable_name: str):
+        random_string = f"batch-ept-{uuid.uuid4().hex[:15]}"
+        return variable_recorder.get_or_record(variable_name, random_string)
+
+    return generate_random_string
+
+@pytest.fixture
+def rand_batch_deployment_name(variable_recorder: VariableRecorder) -> Callable[[str], str]:
+    """return a random batch deployment name string  e.g. batch-dpm-xxx"""
+
+    def generate_random_string(variable_name: str):
+        random_string = f"batch-dpm-{uuid.uuid4().hex[:15]}"
+        return variable_recorder.get_or_record(variable_name, random_string)
+
+    return generate_random_string
+
+@pytest.fixture
+def rand_online_name(variable_recorder: VariableRecorder) -> Callable[[str], str]:
+    """return a random online endpoint name string  e.g. online-ept-xxx"""
+
+    def generate_random_string(variable_name: str):
+        random_string = f"online-ept-{uuid.uuid4().hex[:15]}"
+        return variable_recorder.get_or_record(variable_name, random_string)
+
+    return generate_random_string
+
+@pytest.fixture
+def rand_online_deployment_name(variable_recorder: VariableRecorder) -> Callable[[str], str]:
+    """return a random online deployment name string  e.g. online-dpm-xxx"""
+
+    def generate_random_string(variable_name: str):
+        random_string = f"online-dpm-{uuid.uuid4().hex[:15]}"
+        return variable_recorder.get_or_record(variable_name, random_string)
+
+    return generate_random_string
 
 @pytest.fixture
 def rand_compute_name(variable_recorder: VariableRecorder) -> Callable[[str], str]:
@@ -321,8 +361,8 @@ def batch_endpoint_model(client: MLClient) -> Model:
 
 
 @pytest.fixture
-def light_gbm_model(client: MLClient) -> Model:
-    job_name = "light_gbm_job_" + uuid.uuid4().hex
+def light_gbm_model(client: MLClient, variable_recorder: VariableRecorder) -> Model:
+    job_name = variable_recorder.get_or_record("job_name", "light_gbm_job_" + uuid.uuid4().hex)
     model_name = "lightgbm_predict"  # specified in the mlflow training script
 
     try:

sdk/ml/azure-ai-ml/tests/online_services/e2etests/test_online_deployment.py

@@ -1,4 +1,4 @@
-import uuid
+from typing import Callable
 
 import pytest
 
@@ -10,36 +10,38 @@
 
 
 @pytest.mark.e2etest
+@pytest.mark.usefixtures("recorded_test")
 @pytest.mark.production_experience_test
 class TestOnlineDeployment(AzureRecordedTestCase):
     @pytest.mark.skip(
         reason="Tests failing in internal automation due to lack of quota. Cannot record or run in live mode."
     )
-    def test_online_deployment(self, client: MLClient) -> None:
+    def test_online_deployment(self, client: MLClient, rand_online_name: Callable[[], str], rand_online_deployment_name: Callable[[], str]) -> None:
         endpoint_yaml = "tests/test_configs/deployments/online/simple_online_endpoint_mir.yaml"
         deployment_yaml = "tests/test_configs/deployments/online/online_deployment_1.yaml"
-        name = "online-ept-" + uuid.uuid4().hex[:15]
+        name = rand_online_name("name")
         endpoint = load_online_endpoint(endpoint_yaml)
         endpoint.name = name
 
+        deployment_name = rand_online_deployment_name("deployment_name")
         deployment = load_online_deployment(deployment_yaml)
         deployment.endpoint_name = name
-        deployment.name = "online-dpm-" + uuid.uuid4().hex[:15]
+        deployment.name = deployment_name
 
         # create a endpoint
-        client.online_endpoints.begin_create_or_update(endpoint)
+        client.online_endpoints.begin_create_or_update(endpoint).result()
 
         try:
             # create a deployment
-            client.online_deployments.begin_create_or_update(deployment)
+            client.online_deployments.begin_create_or_update(deployment).result()
             dep = client.online_deployments.get(name=deployment.name, endpoint_name=endpoint.name)
             assert dep.name == deployment.name
 
             deps = client.online_deployments.list(endpoint_name=endpoint.name)
             assert len(list(deps)) > 0
 
             endpoint.traffic = {deployment.name: 100}
-            client.online_endpoints.begin_create_or_update(endpoint)
+            client.online_endpoints.begin_create_or_update(endpoint).result()
             endpoint_updated = client.online_endpoints.get(endpoint.name)
             assert endpoint_updated.traffic[deployment.name] == 100
             client.online_endpoints.invoke(
@@ -52,9 +54,9 @@ def test_online_deployment(self, client: MLClient) -> None:
             client.online_endpoints.begin_delete(name=endpoint.name)
 
     @pytest.mark.skip(reason="Known failure")
-    def test_online_deployment_skip_script_validation(self, client: MLClient,variable_recorder) -> None:
-        online_endpoint_name = variable_recorder.get_or_record("online_endpoint_name", "online-endp" + uuid.uuid4().hex[:15])
-        online_deployment_name = variable_recorder.get_or_record("online_deployment_name", "online-dpm" + uuid.uuid4().hex[:15])
+    def test_online_deployment_skip_script_validation(self, client: MLClient, rand_online_name: Callable[[], str], rand_online_deployment_name: Callable[[], str]) -> None:
+        online_endpoint_name = rand_online_name("online_endpoint_name")
+        online_deployment_name = rand_online_deployment_name("online_deployment_name")
 
         # create an online endpoint
         endpoint = ManagedOnlineEndpoint(