Expose the parameters log_training_metrics and log_validation_loss for object detection and instance segmentation (Azure#31479)

* add log_training_metrics and log_validation_loss params into image_model_settings

* add log params to automl_image_object_detection_base, use enum types, update unit tests

* code reformatting using black

* add log params to docstrings

* remove hardcoded log_training_metrics from unit tests

Author: bhimar

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/automl/image_vertical/image_model_settings.py

@@ -86,6 +86,8 @@ class ImageModelSettingsObjectDetectionSchema(ImageModelSettingsSchema):
         allowed_values=[o.value for o in ValidationMetricType],
         casing_transform=camel_to_snake,
     )
+    log_training_metrics = fields.Str()
+    log_validation_loss = fields.Str()
 
     @post_load
     def make(self, data, **kwargs):

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/automl/image/automl_image_object_detection_base.py

@@ -11,6 +11,8 @@
     ModelSize,
     StochasticOptimizer,
     ValidationMetricType,
+    LogTrainingMetrics,
+    LogValidationLoss,
 )
 from azure.ai.ml._utils.utils import camel_to_snake
 from azure.ai.ml.entities._job.automl import SearchSpace
@@ -56,6 +58,8 @@ def training_parameters(self, value: Union[Dict, ImageModelSettingsObjectDetecti
                 learning_rate_scheduler=value.learning_rate_scheduler,
                 model_size=value.model_size,
                 validation_metric_type=value.validation_metric_type,
+                log_training_metrics=value.log_training_metrics,
+                log_validation_loss=value.log_validation_loss,
             )
         elif value is None:
             self._training_parameters = value
@@ -148,6 +152,8 @@ def set_training_parameters(
         tile_predictions_nms_threshold: Optional[float] = None,
         validation_iou_threshold: Optional[float] = None,
         validation_metric_type: Optional[Union[str, ValidationMetricType]] = None,
+        log_training_metrics: Optional[Union[str, LogTrainingMetrics]] = None,
+        log_validation_loss: Optional[Union[str, LogValidationLoss]] = None,
     ) -> None:
         """Setting Image training parameters for for AutoML Image Object Detection and Image Instance Segmentation
         tasks.
@@ -288,6 +294,12 @@ def set_training_parameters(
         :keyword validation_metric_type: Metric computation method to use for validation metrics. Must
          be 'none', 'coco', 'voc', or 'coco_voc'.
         :type validation_metric_type: str or ~azure.mgmt.machinelearningservices.models.ValidationMetricType
+        :keyword log_training_metrics: indicates whether or not to log training metrics. Must
+         be 'Enable' or 'Disable'
+        :type log_training_metrics: str or ~azure.mgmt.machinelearningservices.models.LogTrainingMetrics
+        :keyword log_validation_loss: indicates whether or not to log validation loss. Must
+         be 'Enable' or 'Disable'
+        :type log_validation_loss: str or ~azure.mgmt.machinelearningservices.models.LogValidationLoss
         """
         self._training_parameters = self._training_parameters or ImageModelSettingsObjectDetection()
 
@@ -432,6 +444,16 @@ def set_training_parameters(
             if validation_metric_type is not None
             else self._training_parameters.validation_metric_type
         )
+        self._training_parameters.log_training_metrics = (
+            LogTrainingMetrics[camel_to_snake(log_training_metrics)]
+            if log_training_metrics is not None
+            else self._training_parameters.log_training_metrics
+        )
+        self._training_parameters.log_validation_loss = (
+            LogValidationLoss[camel_to_snake(log_validation_loss)]
+            if log_validation_loss is not None
+            else self._training_parameters.log_validation_loss
+        )
 
     # pylint: enable=too-many-locals
 

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/automl/image/image_model_settings.py

@@ -16,6 +16,8 @@
     ModelSize,
     StochasticOptimizer,
     ValidationMetricType,
+    LogTrainingMetrics,
+    LogValidationLoss,
 )
 from azure.ai.ml.entities._mixins import RestTranslatableMixin
 
@@ -624,6 +626,12 @@ class ImageModelSettingsObjectDetection(ImageModelDistributionSettings):
      values include: "None", "Coco", "Voc", "CocoVoc".
     :type validation_metric_type: str or
      ~azure.mgmt.machinelearningservices.models.ValidationMetricType
+    :param log_training_metrics: indicates whether or not to log training metrics
+    :type log_training_metrics: str or
+     ~azure.mgmt.machinelearningservices.models.LogTrainingMetrics
+    :param log_validation_loss: indicates whether or not to log validation loss
+    :type log_validation_loss: str or
+     ~azure.mgmt.machinelearningservices.models.LogValidationLoss
     """
 
     def __init__(
@@ -672,6 +680,8 @@ def __init__(
         tile_predictions_nms_threshold: Optional[float] = None,
         validation_iou_threshold: Optional[float] = None,
         validation_metric_type: Optional[ValidationMetricType] = None,
+        log_training_metrics: Optional[LogTrainingMetrics] = None,
+        log_validation_loss: Optional[LogValidationLoss] = None,
         **kwargs,
     ):
         super(ImageModelSettingsObjectDetection, self).__init__(
@@ -720,11 +730,11 @@ def __init__(
         self.tile_predictions_nms_threshold = tile_predictions_nms_threshold
         self.validation_iou_threshold = validation_iou_threshold
         self.validation_metric_type = validation_metric_type
+        self.log_training_metrics = log_training_metrics
+        self.log_validation_loss = log_validation_loss
 
     def _to_rest_object(self) -> RestImageModelSettingsObjectDetection:
         return RestImageModelSettingsObjectDetection(
-            # Temporary fix for https://msdata.visualstudio.com/Vienna/_workitems/edit/2385143
-            log_training_metrics="Disable",
             advanced_settings=self.advanced_settings,
             ams_gradient=self.ams_gradient,
             beta1=self.beta1,
@@ -768,6 +778,8 @@ def _to_rest_object(self) -> RestImageModelSettingsObjectDetection:
             tile_predictions_nms_threshold=self.tile_predictions_nms_threshold,
             validation_iou_threshold=self.validation_iou_threshold,
             validation_metric_type=self.validation_metric_type,
+            log_training_metrics=self.log_training_metrics,
+            log_validation_loss=self.log_validation_loss,
         )
 
     @classmethod
@@ -816,6 +828,8 @@ def _from_rest_object(cls, obj: RestImageModelSettingsObjectDetection) -> "Image
             tile_predictions_nms_threshold=obj.tile_predictions_nms_threshold,
             validation_iou_threshold=obj.validation_iou_threshold,
             validation_metric_type=obj.validation_metric_type,
+            log_training_metrics=obj.log_training_metrics,
+            log_validation_loss=obj.log_validation_loss,
         )
 
     def __eq__(self, other: object) -> bool:
@@ -837,6 +851,8 @@ def __eq__(self, other: object) -> bool:
             and self.tile_predictions_nms_threshold == other.tile_predictions_nms_threshold
             and self.validation_iou_threshold == other.validation_iou_threshold
             and self.validation_metric_type == other.validation_metric_type
+            and self.log_training_metrics == other.log_training_metrics
+            and self.log_validation_loss == other.log_validation_loss
         )
 
     def __ne__(self, other: object) -> bool:

sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_instance_segmentation.py

@@ -12,6 +12,8 @@
     ModelSize,
     SamplingAlgorithmType,
     StochasticOptimizer,
+    LogTrainingMetrics,
+    LogValidationLoss,
 )
 from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity
 from azure.ai.ml._restclient.v2023_04_01_preview.models import ValidationMetricType
@@ -128,24 +130,38 @@ def _check_data_type(data, expected_type, expected_path, msg):
         "settings, expected",
         [
             (
-                ("adam", "warmup_cosine", "coco_voc", "large"),
+                ("adam", "warmup_cosine", "coco_voc", "large", "enable", "enable"),
                 (
                     StochasticOptimizer.ADAM,
                     LearningRateScheduler.WARMUP_COSINE,
                     ValidationMetricType.COCO_VOC,
                     ModelSize.LARGE,
+                    LogTrainingMetrics.ENABLE,
+                    LogValidationLoss.ENABLE,
                 ),
             ),
             (
-                ("Adam", "WarmupCosine", "CocoVoc", "Large"),
+                ("Adam", "WarmupCosine", "CocoVoc", "Large", "Enable", "Enable"),
                 (
                     StochasticOptimizer.ADAM,
                     LearningRateScheduler.WARMUP_COSINE,
                     ValidationMetricType.COCO_VOC,
                     ModelSize.LARGE,
+                    LogTrainingMetrics.ENABLE,
+                    LogValidationLoss.ENABLE,
+                ),
+            ),
+            (
+                (None, None, "coco_voc", "large", "enable", "enable"),
+                (
+                    None,
+                    None,
+                    ValidationMetricType.COCO_VOC,
+                    ModelSize.LARGE,
+                    LogTrainingMetrics.ENABLE,
+                    LogValidationLoss.ENABLE,
                 ),
             ),
-            ((None, None, "coco_voc", "large"), (None, None, ValidationMetricType.COCO_VOC, ModelSize.LARGE)),
         ],
         ids=["snake case", "camel case", "with None"],
     )
@@ -159,25 +175,33 @@ def test_image_set_training_parameters_with_valid_values(self, settings, expecte
             learning_rate_scheduler=settings[1],
             validation_metric_type=settings[2],
             model_size=settings[3],
+            log_training_metrics=settings[4],
+            log_validation_loss=settings[5],
         )
         assert image_instance_segmentation_job.training_parameters.optimizer == expected[0]
         assert image_instance_segmentation_job.training_parameters.learning_rate_scheduler == expected[1]
         assert image_instance_segmentation_job.training_parameters.validation_metric_type == expected[2]
         assert image_instance_segmentation_job.training_parameters.model_size == expected[3]
+        assert image_instance_segmentation_job.training_parameters.log_training_metrics == expected[4]
+        assert image_instance_segmentation_job.training_parameters.log_validation_loss == expected[5]
 
     @pytest.mark.parametrize(
         "settings, expected",
         [
-            (("adamW", None, None, None), pytest.raises(KeyError)),
-            ((None, "Warmup_Cosine", None, None), pytest.raises(KeyError)),
-            ((None, None, "Coco_Voc", "large"), pytest.raises(KeyError)),
-            ((None, None, None, "Extra_Large"), pytest.raises(KeyError)),
+            (("adamW", None, None, None, "Enable", "Enable"), pytest.raises(KeyError)),
+            ((None, "Warmup_Cosine", None, None, "Enable", "Enable"), pytest.raises(KeyError)),
+            ((None, None, "Coco_Voc", "large", "Enable", "Enable"), pytest.raises(KeyError)),
+            ((None, None, None, "Extra_Large", "Enable", "Enable"), pytest.raises(KeyError)),
+            ((None, None, None, None, "false", "Enable"), pytest.raises(KeyError)),
+            ((None, None, None, None, "Enable", "false"), pytest.raises(KeyError)),
         ],
         ids=[
             "optimizer invalid",
             "learning rate scheduler invalid",
             "validation metric type invalid",
             "model size invalid",
+            "log_training_metrics invalid",
+            "log_validation_loss invalid",
         ],
     )
     def test_image_set_training_parameters_with_invalid_values(self, settings, expected):
@@ -191,30 +215,46 @@ def test_image_set_training_parameters_with_invalid_values(self, settings, expec
                 learning_rate_scheduler=settings[1],
                 validation_metric_type=settings[2],
                 model_size=settings[3],
+                log_training_metrics=settings[4],
+                log_validation_loss=settings[5],
             )
 
     @pytest.mark.parametrize(
         "settings, expected",
         [
             (
-                ("adam", "warmup_cosine", "coco_voc", "large"),
+                ("adam", "warmup_cosine", "coco_voc", "large", "enable", "enable"),
                 (
                     StochasticOptimizer.ADAM,
                     LearningRateScheduler.WARMUP_COSINE,
                     ValidationMetricType.COCO_VOC,
                     ModelSize.LARGE,
+                    LogTrainingMetrics.ENABLE,
+                    LogValidationLoss.ENABLE,
                 ),
             ),
             (
-                ("Adam", "WarmupCosine", "CocoVoc", "Large"),
+                ("Adam", "WarmupCosine", "CocoVoc", "Large", "Enable", "Enable"),
                 (
                     StochasticOptimizer.ADAM,
                     LearningRateScheduler.WARMUP_COSINE,
                     ValidationMetricType.COCO_VOC,
                     ModelSize.LARGE,
+                    LogTrainingMetrics.ENABLE,
+                    LogValidationLoss.ENABLE,
+                ),
+            ),
+            (
+                (None, None, "coco_voc", "large", "enable", "enable"),
+                (
+                    None,
+                    None,
+                    ValidationMetricType.COCO_VOC,
+                    ModelSize.LARGE,
+                    LogTrainingMetrics.ENABLE,
+                    LogValidationLoss.ENABLE,
                 ),
             ),
-            ((None, None, "coco_voc", "large"), (None, None, ValidationMetricType.COCO_VOC, ModelSize.LARGE)),
         ],
         ids=["snake case", "camel case", "with None"],
     )
@@ -224,13 +264,18 @@ def test_image_set_training_parameters_with_settings_object(self, settings, expe
             learning_rate_scheduler=settings[1],
             validation_metric_type=settings[2],
             model_size=settings[3],
+            log_training_metrics=settings[4],
+            log_validation_loss=settings[5],
         )
         image_instance_segmentation_job = image_instance_segmentation(
             training_data=Input(type=AssetTypes.MLTABLE, path="https://foo/bar/train.csv"),
             target_column_name="label",
             training_parameters=image_model_settings,
         )
+
         assert image_instance_segmentation_job.training_parameters.optimizer == expected[0]
         assert image_instance_segmentation_job.training_parameters.learning_rate_scheduler == expected[1]
         assert image_instance_segmentation_job.training_parameters.validation_metric_type == expected[2]
         assert image_instance_segmentation_job.training_parameters.model_size == expected[3]
+        assert image_instance_segmentation_job.training_parameters.log_training_metrics == expected[4]
+        assert image_instance_segmentation_job.training_parameters.log_validation_loss == expected[5]