Adding feature set operations for list materialization, backfill and get feature (Azure#29270)

* Adding tags for feature, and feature set operations

* Adding unit tests for feature set operations

* Lint fix

* Update feature schema

* Updating properties of Metadata schema based on the service properties name

* update to_dict to create  dictionary based on service properties

* Adding deserialize callback for feature set and feature store entity

* fix black

* update properties name

* Updating properties and schema metadata class

* update progress

* Remove kind property from worksace for release

* Remove kind from constructor

* Default kind for workspace

* Update Default to default

* Update get to pop method

* update progress

* Add _ prefix in load functions for feature store

* Adding feature store classes for private preview

* Fix Feature transformation code schema

* Fix names for feature set operations

* Removing feature set operations and load functions

* Remove from init files

* Removing experimental for feature store operation classes

* import operations from nested path in ml client

* Remove load functions from init files

* Fix tests

* fix import in feature set test

* Remove properties from feature store entity constructor

* update feature_schema

Author: maykumar-msft

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_feature_set/__init__.py

@@ -7,7 +7,7 @@
 from .delay_metadata_schema import DelayMetadataSchema
 from .feature_schema import FeatureSchema
 from .feature_set_schema import FeatureSetSchema
-from .featureset_spec_schema import FeaturesetSpecSchema
+from .featureset_spec_metadata_schema import FeaturesetSpecMetadataSchema
 from .feature_set_specification_schema import FeatureSetSpecificationSchema
 from .materialization_settings_schema import MaterializationSettingsSchema
 from .source_metadata_schema import SourceMetadataSchema
@@ -17,7 +17,7 @@
     "DelayMetadataSchema",
     "FeatureSchema",
     "FeatureSetSchema",
-    "FeaturesetSpecSchema",
+    "FeaturesetSpecMetadataSchema",
     "FeatureSetSpecificationSchema",
     "MaterializationSettingsSchema",
     "SourceMetadataSchema",

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_feature_set/feature_schema.py

@@ -14,15 +14,17 @@ class FeatureSchema(metaclass=PatchedSchemaMeta):
         required=True,
         allow_none=False,
     )
-    type = fields.Str(
+    data_type = fields.Str(
         validate=validate.OneOf(["string", "integer", "long", "float", "double", "binary", "datetime", "boolean"]),
         required=True,
         allow_none=False,
+        data_key="type",
     )
     description = fields.Str(required=False)
+    tags = fields.Dict(keys=fields.Str(), values=fields.Str(), required=False)
 
     @post_load
     def make(self, data, **kwargs):
         from azure.ai.ml.entities._feature_set.feature import _Feature
 
-        return _Feature(data_type=type, description=data.pop("description", None), **data)
+        return _Feature(description=data.pop("description", None), **data)

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_feature_set/feature_set_schema.py

@@ -21,5 +21,4 @@ class FeatureSetSchema(YamlFileSchema):
     stage = fields.Str()
     description = fields.Str()
     tags = fields.Dict(keys=fields.Str(), values=fields.Str())
-    properties = fields.Dict(keys=fields.Str(), values=fields.Str())
     materialization_settings = NestedField(MaterializationSettingsSchema)

…et/feature_transformation_code_schema.py …e_transformation_code_metadata_schema.pysdk/ml/azure-ai-ml/azure/ai/ml/_schema/_feature_set/feature_transformation_code_schema.py renamed to sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_feature_set/feature_transformation_code_metadata_schema.py sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_feature_set/feature_transformation_code_schema.py renamed to sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_feature_set/feature_transformation_code_metadata_schema.py

@@ -9,12 +9,14 @@
 from azure.ai.ml._schema.core.schema import PatchedSchemaMeta
 
 
-class FeatureTransformationCodeSchema(metaclass=PatchedSchemaMeta):
+class FeatureTransformationCodeMetadataSchema(metaclass=PatchedSchemaMeta):
     path = fields.Str(required=False)
     transformer_class = fields.Str(required=False)
 
     @post_load
     def make(self, data, **kwargs):
-        from azure.ai.ml.entities._feature_set.feature_transformation_code import FeatureTransformationCode
+        from azure.ai.ml.entities._feature_set.feature_transformation_code_metadata import (
+            FeatureTransformationCodeMetadata,
+        )
 
-        return FeatureTransformationCode(**data)
+        return FeatureTransformationCodeMetadata(**data)

…a/_feature_set/featureset_spec_schema.py …e_set/featureset_spec_metadata_schema.pysdk/ml/azure-ai-ml/azure/ai/ml/_schema/_feature_set/featureset_spec_schema.py renamed to sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_feature_set/featureset_spec_metadata_schema.py sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_feature_set/featureset_spec_schema.py renamed to sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_feature_set/featureset_spec_metadata_schema.py

@@ -15,19 +15,19 @@
 from .source_metadata_schema import SourceMetadataSchema
 from .delay_metadata_schema import DelayMetadataSchema
 from .feature_schema import FeatureSchema
-from .feature_transformation_code_schema import FeatureTransformationCodeSchema
+from .feature_transformation_code_metadata_schema import FeatureTransformationCodeMetadataSchema
 
 
-class FeaturesetSpecSchema(YamlFileSchema):
+class FeaturesetSpecMetadataSchema(YamlFileSchema):
     source = fields.Nested(SourceMetadataSchema, required=True)
-    feature_transformation_code = fields.Nested(FeatureTransformationCodeSchema, required=False)
+    feature_transformation_code = fields.Nested(FeatureTransformationCodeMetadataSchema, required=False)
     features = fields.List(NestedField(FeatureSchema), required=True, allow_none=False)
     index_columns = fields.List(NestedField(DataColumnSchema), required=True, allow_none=False)
     source_lookback = fields.Nested(DelayMetadataSchema, required=False)
     temporal_join_lookback = fields.Nested(DelayMetadataSchema, required=False)
 
     @post_load
     def make(self, data: Dict, **kwargs):
-        from azure.ai.ml.entities._feature_set.featureset_spec import FeaturesetSpec
+        from azure.ai.ml.entities._feature_set.featureset_spec_metadata import FeaturesetSpecMetadata
 
-        return FeaturesetSpec(**data)
+        return FeaturesetSpecMetadata(**data)

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_feature_set/featureset_spec_properties_schema.py

@@ -0,0 +1,43 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# pylint: disable=unused-argument,no-self-use
+
+
+from marshmallow import fields
+
+from azure.ai.ml._schema.core.fields import NestedField
+from azure.ai.ml._schema.core.schema import YamlFileSchema, PatchedSchemaMeta
+from azure.ai.ml._schema._feature_store_entity.data_column_schema import DataColumnSchema
+
+from .source_metadata_schema import SourceMetadataSchema
+
+
+class FeatureTransformationCodePropertiesSchema(metaclass=PatchedSchemaMeta):
+    path = fields.Str(data_key="Path")
+    transformer_class = fields.Str(data_key="TransformerClass")
+
+
+class DelayMetadataPropertiesSchema(metaclass=PatchedSchemaMeta):
+    days = fields.Int(data_key="Days")
+    hours = fields.Int(data_key="Hours")
+    minutes = fields.Int(data_key="Minutes")
+
+
+class FeaturePropertiesSchema(metaclass=PatchedSchemaMeta):
+    name = fields.Str(data_key="FeatureName")
+    data_type = fields.Str(data_key="DataType")
+    description = fields.Str(data_key="Description")
+    tags = fields.Dict(keys=fields.Str(), values=fields.Str(), data_key="Tags")
+
+
+class FeaturesetSpecPropertiesSchema(YamlFileSchema):
+    source = fields.Nested(SourceMetadataSchema, data_key="source")
+    feature_transformation_code = fields.Nested(
+        FeatureTransformationCodePropertiesSchema, data_key="featureTransformationCode"
+    )
+    features = fields.List(NestedField(FeaturePropertiesSchema), data_key="features")
+    index_columns = fields.List(NestedField(DataColumnSchema), data_key="indexColumns")
+    source_lookback = fields.Nested(DelayMetadataPropertiesSchema, data_key="sourceLookback")
+    temporal_join_lookback = fields.Nested(DelayMetadataPropertiesSchema, data_key="temporalJoinLookback")

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_assets/_artifacts/feature_set.py

@@ -44,7 +44,6 @@ def __init__(
         description: Optional[str] = None,
         materialization_settings: Optional[_MaterializationSettings] = None,
         tags: Optional[Dict] = None,
-        properties: Optional[Dict[str, str]] = None,
         **kwargs,
     ):
         """FeatureSet
@@ -61,8 +60,6 @@ def __init__(
         :type description: str
         :param tags: Tag dictionary. Tags can be added, removed, and updated.
         :type tags: dict[str, str]
-        :param properties: The asset property dictionary.
-        :type properties: dict[str, str]
         :param materialization_settings: Specifies the materialization settings.
         :type materialization_settings: ~azure.ai.ml.entities._MaterializationSettings
         :param kwargs: A dictionary of additional configuration parameters.
@@ -73,7 +70,6 @@ def __init__(
             version=version,
             description=description,
             tags=tags,
-            properties=properties,
             path=specification.path,
             **kwargs,
         )
@@ -109,7 +105,6 @@ def _from_rest_object(cls, featureset_rest_object: FeaturesetVersion) -> "_Featu
             version=arm_id_object.asset_version,
             description=featureset_rest_object_details.description,
             tags=featureset_rest_object_details.tags,
-            properties=featureset_rest_object_details.properties,
             entities=featureset_rest_object_details.entities,
             materialization_settings=_MaterializationSettings._from_rest_object(
                 featureset_rest_object_details.materialization_settings
@@ -127,7 +122,6 @@ def _from_container_rest_object(cls, rest_obj: FeaturesetContainer) -> "_Feature
             name=arm_id_object.asset_name,
             description=rest_object_details.description,
             tags=rest_object_details.tags,
-            properties=rest_object_details.properties,
             entities=[],
             specification=_FeatureSetSpecification(),
             version="",

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_feature_set/feature.py

@@ -2,14 +2,40 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-# pylint: disable=unused-argument
+# pylint: disable=redefined-builtin,disable=unused-argument
 
-from typing import Optional
+from typing import Dict, Optional
+
+from azure.ai.ml._restclient.v2023_02_01_preview.models import (
+    Feature as RestFeature,
+)
+
+from azure.ai.ml.entities._mixins import RestTranslatableMixin
 from azure.ai.ml.entities._feature_store_entity.data_column_type import _DataColumnType
+from azure.ai.ml._utils._experimental import experimental
 
 
-class _Feature(object):
-    def __init__(self, *, name: str, data_type: _DataColumnType, description: Optional[str], **kwargs):
+@experimental
+class _Feature(RestTranslatableMixin):
+    def __init__(
+        self,
+        *,
+        name: str,
+        data_type: _DataColumnType,
+        description: Optional[str] = None,
+        tags: Optional[Dict[str, str]] = None,
+        **kwargs
+    ):
         self.name = name
-        self.type = data_type
+        self.data_type = data_type
         self.description = description
+        self.tags = tags
+
+    @classmethod
+    def _from_rest_object(cls, obj: RestFeature) -> "_Feature":
+        return _Feature(
+            name=obj.feature_name,
+            data_type=obj.data_type,
+            description=obj.description,
+            tags=obj.tags,
+        )

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_feature_set/feature_set_materialization_response.py

@@ -0,0 +1,63 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from datetime import datetime, timedelta
+from typing import Dict, Optional, Union
+
+from azure.ai.ml._restclient.v2023_02_01_preview.models import (
+    FeaturesetJob as RestFeaturesetJob,
+)
+from azure.ai.ml.entities._mixins import RestTranslatableMixin
+from azure.ai.ml.entities._system_data import SystemData
+from azure.ai.ml._utils._experimental import experimental
+
+from .materialization_type import _MaterializationType
+
+FeaturestoreJobTypeMap: Dict[str, _MaterializationType] = {
+    "BackfillMaterialization": _MaterializationType.BackfillMaterialization,
+    "RecurrentMaterialization": _MaterializationType.RecurrentMaterialization,
+}
+
+
+@experimental
+class _FeatureSetMaterializationResponse(RestTranslatableMixin):
+    def __init__(
+        self,
+        *,
+        type: Union[_MaterializationType, str],  # pylint: disable=redefined-builtin
+        feature_window_start_time: Optional[Union[str, datetime]],
+        feature_window_end_time: Optional[Union[str, datetime]],
+        name: Optional[str],
+        display_name: Optional[str],
+        creation_context: Optional[SystemData],
+        duration: Optional[Union[str, timedelta]],
+        status: Optional[str],
+        tags: Optional[Dict[str, str]],
+        **kwargs  # pylint: disable=unused-argument
+    ):
+        self.type = type
+        self.feature_window_start_time = feature_window_start_time
+        self.feature_window_end_time = feature_window_end_time
+        self.name = name
+        self.display_name = display_name
+        self.creation_context = creation_context
+        self.duration = duration
+        self.status = status
+        self.tags = tags
+
+    @classmethod
+    def _from_rest_object(cls, obj: RestFeaturesetJob) -> "_FeatureSetMaterializationResponse":
+        if not obj:
+            return None
+        return _FeatureSetMaterializationResponse(
+            type=FeaturestoreJobTypeMap.get(obj.type, obj.type),
+            feature_window_start_time=obj.feature_window.feature_window_start,
+            feature_window_end_time=obj.feature_window.feature_window_end,
+            name=obj.job_id,
+            display_name=obj.display_name,
+            creation_context=SystemData(created_at=obj.created_date),
+            duration=obj.duration,
+            status=obj.status,
+            tags=obj.tags,
+        )

…ature_set/feature_transformation_code.py …/feature_transformation_code_metadata.pysdk/ml/azure-ai-ml/azure/ai/ml/entities/_feature_set/feature_transformation_code.py renamed to sdk/ml/azure-ai-ml/azure/ai/ml/entities/_feature_set/feature_transformation_code_metadata.py sdk/ml/azure-ai-ml/azure/ai/ml/entities/_feature_set/feature_transformation_code.py renamed to sdk/ml/azure-ai-ml/azure/ai/ml/entities/_feature_set/feature_transformation_code_metadata.py

@@ -7,7 +7,7 @@
 from typing import Optional
 
 
-class FeatureTransformationCode(object):
+class FeatureTransformationCodeMetadata(object):
     def __init__(self, *, path: str, transformer_class: Optional[str] = None, **kwargs):
         self.path = path
         self.transformer_class = transformer_class