Skip to content

Commit f182f7d

Browse files
TajiHarrisMicrosoftTajiHarrisMicrosoft
authored
payload logging model data collector development (Azure#28491)
* payload logging model data collector development * resolving pr comments * implement collections and data asset to schema * implement nested collections * resolving event hub yaml errors * resolve pylint issues * add version to data asset description * resolve spelling errors and add exclusive numeric notation
1 parent 042275f commit f182f7d

18 files changed

+250
-118
lines changed

‎sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_deployment/online/data_asset_schema.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# ---------------------------------------------------------
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# ---------------------------------------------------------
4+
5+
# pylint: disable=no-self-use
6+
7+
import logging
8+
from typing import Any
9+
10+
from marshmallow import fields, post_load
11+
12+
from azure.ai.ml._schema import PatchedSchemaMeta
13+
14+
module_logger = logging.getLogger(__name__)
15+
16+
17+
class DataAssetSchema(metaclass=PatchedSchemaMeta):
18+
name = fields.Str()
19+
path = fields.Str()
20+
version = fields.Int()
21+
22+
@post_load
23+
def make(self, data: Any, **kwargs: Any) -> Any: # pylint: disable=unused-argument
24+
from azure.ai.ml.entities._deployment.data_asset import DataAsset
25+
26+
return DataAsset(**data)

‎sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_deployment/online/data_collector_schema.py‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,27 +7,34 @@
77
import logging
88
from typing import Any
99

10-
from marshmallow import fields, post_load
10+
from marshmallow import fields, post_load, validates, ValidationError
1111

1212
from azure.ai.ml._schema import NestedField, PatchedSchemaMeta, StringTransformedEnum
1313
from azure.ai.ml._schema._deployment.online.destination_schema import DestinationSchema
1414
from azure.ai.ml._schema._deployment.online.request_logging_schema import RequestLoggingSchema
15-
from azure.ai.ml._schema._deployment.online.sampling_strategy_schema import SamplingStrategySchema
15+
from azure.ai.ml._schema._deployment.online.deployment_collection_schema import DeploymentCollectionSchema
16+
1617
from azure.ai.ml.constants._common import RollingRate
1718

1819
module_logger = logging.getLogger(__name__)
1920

2021

2122
class DataCollectorSchema(metaclass=PatchedSchemaMeta):
22-
enabled = fields.Bool()
23+
collections = fields.Mapping(fields.Str, NestedField(DeploymentCollectionSchema))
2324
rolling_rate = StringTransformedEnum(
2425
required=False,
25-
allowed_values=[RollingRate.HOUR, RollingRate.YEAR, RollingRate.MONTH, RollingRate.DAY, RollingRate.MINUTE],
26+
allowed_values=[ RollingRate.MINUTE, RollingRate.DAY, RollingRate.HOUR],
2627
)
2728
destination = NestedField(DestinationSchema)
28-
sampling_strategy = NestedField(SamplingStrategySchema)
29+
sampling_rate = fields.Float()
2930
request_logging = NestedField(RequestLoggingSchema)
3031

32+
# pylint: disable=unused-argument,no-self-use
33+
@validates("sampling_rate")
34+
def validate_sampling_rate(self, value, **kwargs):
35+
if value > 1.0 or value < 0.0:
36+
raise ValidationError("Sampling rate must be an number in range (0.0-1.0)")
37+
3138
@post_load
3239
def make(self, data: Any, **kwargs: Any) -> Any: # pylint: disable=unused-argument
3340
from azure.ai.ml.entities._deployment.data_collector import DataCollector

‎sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_deployment/online/deployment_collection_schema.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# ---------------------------------------------------------
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# ---------------------------------------------------------
4+
5+
import logging
6+
from typing import Any
7+
8+
from marshmallow import post_load
9+
10+
from azure.ai.ml._schema import PatchedSchemaMeta, StringTransformedEnum, NestedField
11+
from azure.ai.ml._schema._deployment.online.data_asset_schema import DataAssetSchema
12+
from azure.ai.ml.constants._common import Boolean
13+
14+
module_logger = logging.getLogger(__name__)
15+
16+
17+
class DeploymentCollectionSchema(metaclass=PatchedSchemaMeta):
18+
enabled = StringTransformedEnum(
19+
required= True,
20+
allowed_values=[Boolean.TRUE, Boolean.FALSE]
21+
)
22+
data = NestedField(DataAssetSchema)
23+
24+
# pylint: disable=unused-argument,no-self-use
25+
@post_load
26+
def make(self, data: Any, **kwargs: Any) -> Any:
27+
from azure.ai.ml.entities._deployment.deployment_collection import DeploymentCollection
28+
29+
return DeploymentCollection(**data)

‎sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_deployment/online/event_hub_schema.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
class EventHubSchema(metaclass=PatchedSchemaMeta):
1919
namespace = fields.Str()
2020
oversize_data_config = NestedField(OversizeDataConfigSchema)
21-
client_id = fields.Str()
2221

2322
@validates("namespace")
2423
def validate_namespace(self, value, **kwargs):

‎sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_deployment/online/online_deployment.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ class OnlineDeploymentSchema(DeploymentSchema):
4848
)
4949
model_mount_path = fields.Str()
5050
instance_type = fields.Str()
51+
data_collector = ExperimentalField(NestedField(DataCollectorSchema))
5152

5253

5354
class KubernetesOnlineDeploymentSchema(OnlineDeploymentSchema):
@@ -66,7 +67,6 @@ class ManagedOnlineDeploymentSchema(OnlineDeploymentSchema):
6667
egress_public_network_access = StringTransformedEnum(
6768
allowed_values=[PublicNetworkAccess.ENABLED, PublicNetworkAccess.DISABLED]
6869
)
69-
data_collector = ExperimentalField(NestedField(DataCollectorSchema))
7070
private_network_connection = ExperimentalField(fields.Bool())
7171

7272
@post_load

‎sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_deployment/online/oversize_data_config_schema.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515

1616
class OversizeDataConfigSchema(metaclass=PatchedSchemaMeta):
1717
path = fields.Str()
18-
client_id = fields.Str()
1918

2019
# pylint: disable=unused-argument,no-self-use
2120
@validates("path")

‎sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_deployment/online/payload_response_schema.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# ---------------------------------------------------------
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# ---------------------------------------------------------
4+
5+
import logging
6+
from typing import Any
7+
8+
from marshmallow import post_load
9+
10+
from azure.ai.ml._schema import PatchedSchemaMeta, StringTransformedEnum
11+
from azure.ai.ml.constants._common import Boolean
12+
13+
module_logger = logging.getLogger(__name__)
14+
15+
16+
class PayloadResponseSchema(metaclass=PatchedSchemaMeta):
17+
enabled = StringTransformedEnum(
18+
required= True,
19+
allowed_values=[Boolean.TRUE, Boolean.FALSE]
20+
)
21+
22+
# pylint: disable=unused-argument,no-self-use
23+
@post_load
24+
def make(self, data: Any, **kwargs: Any) -> Any:
25+
from azure.ai.ml.entities._deployment.payload_response import PayloadResponse
26+
27+
return PayloadResponse(**data)

‎sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_deployment/online/sampling_strategy_schema.py‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎sdk/ml/azure-ai-ml/azure/ai/ml/constants/_common.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -581,8 +581,6 @@ class ModelType:
581581

582582

583583
class RollingRate:
584-
YEAR = "year"
585-
MONTH = "month"
586584
DAY = "day"
587585
HOUR = "hour"
588586
MINUTE = "minute"
@@ -597,3 +595,7 @@ class IdentityType:
597595
AML_TOKEN = "aml_token"
598596
USER_IDENTITY = "user_identity"
599597
MANAGED_IDENTITY = "managed_identity"
598+
599+
class Boolean:
600+
TRUE = "true"
601+
FALSE = "false"

‎sdk/ml/azure-ai-ml/azure/ai/ml/entities/_deployment/data_asset.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# ---------------------------------------------------------
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# ---------------------------------------------------------
4+
5+
from typing import Any, Dict, Optional
6+
7+
from azure.ai.ml._schema._deployment.online.data_asset_schema import DataAssetSchema
8+
from azure.ai.ml._utils._experimental import experimental
9+
from azure.ai.ml.constants._common import BASE_PATH_CONTEXT_KEY
10+
11+
12+
@experimental
13+
class DataAsset:
14+
"""Data Asset entity
15+
16+
:param name: Name of data asset
17+
:type name: str
18+
:param path: Path where the data asset is stored.
19+
:type path: str
20+
:param version: Version of data asset.
21+
:type version" int
22+
23+
"""
24+
25+
def __init__(
26+
self,
27+
name: Optional[str] = None,
28+
path: Optional[str] = None,
29+
version: Optional[int] = None,
30+
**kwargs,
31+
): # pylint: disable=unused-argument
32+
self.name = name
33+
self.path = path
34+
self.version = version
35+
36+
def _to_dict(self) -> Dict:
37+
# pylint: disable=no-member
38+
return DataAssetSchema(context={BASE_PATH_CONTEXT_KEY: "./"}).dump(self)

0 commit comments

Comments
 (0)