Revert "[ML][Pipelines] feat: load component from flow (Azure#31632) (Azure#31911)

This reverts commit 53ab388.

Author: scbedd

.vscode/cspell.json

@@ -88,7 +88,6 @@
     "sdk/ml/azure-ai-ml/tests/**",
     "sdk/ml/azure-ai-ml/swagger/**",
     "sdk/ml/azure-ai-ml/NOTICE.txt",
-    "sdk/ml/azure-ai-ml/.pre-commit-config.yaml",
     "sdk/loadtestservice/azure-developer-loadtesting/**",
     "sdk/translation/azure-ai-translation-text/azure/ai/translation/text/_serialization.py",
     "sdk/translation/azure-ai-translation-text/tests/test_break_sentence.py",

sdk/ml/azure-ai-ml/.pre-commit-config.yaml

@@ -26,8 +26,8 @@ repos:
     args: ['--settings-file', 'sdk/ml/azure-ai-ml/pyproject.toml']
 - repo: local
   hooks:
-    - id: pylint-dependencies-check
-      name: pylint-dependencies-check
+    - id: pylint
+      name: pylint
       entry: python
       language: system
       types: [python]
@@ -69,9 +69,9 @@ repos:
             print(f"Please run the following command to install the correct version of {packagename}")
             print(f"\tpython -m pip install {packagename}=={required_version} {' '.join(install_args)}")
             sys.exit(1)
-    - id: pylint
-      name: pylint
-      entry: pylint
-      language: system
-      args: [--rcfile=pylintrc, --output-format=parseable]
-      files: '^sdk/ml/azure-ai-ml/azure/ai/ml/.*\.py$'
+
+        # Run pylint
+        os.execl(sys.executable, sys.executable, "-m", "pylint", *sys.argv[1:])
+      # cspell:disable-next-line
+      - "--rcfile=pylintrc"
+      - "--output-format=parseable"

sdk/ml/azure-ai-ml/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/ml/azure-ai-ml",
-  "Tag": "python/ml/azure-ai-ml_b8d9d78843"
+  "Tag": "python/ml/azure-ai-ml_d05969c2b4"
 }

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/component/__init__.py

@@ -5,21 +5,21 @@
 __path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: ignore
 
 from .command_component import AnonymousCommandComponentSchema, CommandComponentSchema, ComponentFileRefField
-from .component import ComponentSchema, ComponentYamlRefField
+from .component import ComponentSchema
+from .import_component import AnonymousImportComponentSchema, ImportComponentFileRefField, ImportComponentSchema
+from .parallel_component import AnonymousParallelComponentSchema, ParallelComponentFileRefField, ParallelComponentSchema
+from .spark_component import AnonymousSparkComponentSchema, SparkComponentFileRefField, SparkComponentSchema
 from .data_transfer_component import (
     AnonymousDataTransferCopyComponentSchema,
-    AnonymousDataTransferExportComponentSchema,
-    AnonymousDataTransferImportComponentSchema,
     DataTransferCopyComponentFileRefField,
     DataTransferCopyComponentSchema,
-    DataTransferExportComponentFileRefField,
-    DataTransferExportComponentSchema,
+    AnonymousDataTransferImportComponentSchema,
     DataTransferImportComponentFileRefField,
     DataTransferImportComponentSchema,
+    AnonymousDataTransferExportComponentSchema,
+    DataTransferExportComponentFileRefField,
+    DataTransferExportComponentSchema,
 )
-from .import_component import AnonymousImportComponentSchema, ImportComponentFileRefField, ImportComponentSchema
-from .parallel_component import AnonymousParallelComponentSchema, ParallelComponentFileRefField, ParallelComponentSchema
-from .spark_component import AnonymousSparkComponentSchema, SparkComponentFileRefField, SparkComponentSchema
 
 __all__ = [
     "ComponentSchema",
@@ -44,5 +44,4 @@
     "AnonymousDataTransferExportComponentSchema",
     "DataTransferExportComponentFileRefField",
     "DataTransferExportComponentSchema",
-    "ComponentYamlRefField",
 ]

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/component/component.py

@@ -1,10 +1,8 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from pathlib import Path
 
-from marshmallow import ValidationError, fields, post_dump, pre_dump, pre_load
-from marshmallow.fields import Field
+from marshmallow import fields, post_dump, pre_dump, pre_load
 
 from azure.ai.ml._schema.component.input_output import InputPortSchema, OutputPortSchema, ParameterSchema
 from azure.ai.ml._schema.core.fields import (
@@ -15,8 +13,8 @@
     UnionField,
 )
 from azure.ai.ml._schema.core.intellectual_property import IntellectualPropertySchema
-from azure.ai.ml._utils.utils import is_private_preview_enabled, load_yaml
-from azure.ai.ml.constants._common import BASE_PATH_CONTEXT_KEY, AzureMLResourceType
+from azure.ai.ml._utils.utils import is_private_preview_enabled
+from azure.ai.ml.constants._common import AzureMLResourceType
 
 from .._utils.utils import _resolve_group_inputs_for_component
 from ..assets.asset import AssetSchema
@@ -28,52 +26,6 @@ def _get_field_name(self):
         return "Component"
 
 
-class ComponentYamlRefField(Field):
-    """Allows you to nest a :class:`Schema <marshmallow.Schema>`
-    inside a yaml ref field.
-    """
-
-    def _jsonschema_type_mapping(self):
-        schema = {"type": "string"}
-        if self.name is not None:
-            schema["title"] = self.name
-        if self.dump_only:
-            schema["readonly"] = True
-        return schema
-
-    def _deserialize(self, value, attr, data, **kwargs):
-        if not isinstance(value, str):
-            raise ValidationError(f"Nested yaml ref field expected a string but got {type(value)}.")
-
-        base_path = Path(self.context[BASE_PATH_CONTEXT_KEY])
-
-        source_path = Path(value)
-        # raise if the string is not a valid path, like "azureml:xxx"
-        try:
-            source_path.resolve()
-        except OSError as ex:
-            raise ValidationError(f"Nested file ref field expected a local path but got {value}.") from ex
-
-        if not source_path.is_absolute():
-            source_path = base_path / source_path
-
-        if not source_path.is_file():
-            raise ValidationError(
-                f"Nested yaml ref field expected a local path but can't find {value} based on {base_path.as_posix()}."
-            )
-
-        loaded_value = load_yaml(source_path)
-
-        # local import to avoid circular import
-        from azure.ai.ml.entities import Component
-
-        component = Component._load(data=loaded_value, yaml_path=source_path)  # pylint: disable=protected-access
-        return component
-
-    def _serialize(self, value, attr, obj, **kwargs):
-        raise ValidationError("Serialize on RefField is not supported.")
-
-
 class ComponentSchema(AssetSchema):
     schema = fields.Str(data_key="$schema", attribute="_schema")
     name = ComponentNameStr(required=True)

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/component/flow.py

@@ -15,7 +15,6 @@
     AnonymousParallelComponentSchema,
     AnonymousSparkComponentSchema,
     ComponentFileRefField,
-    ComponentYamlRefField,
     DataTransferCopyComponentFileRefField,
     ImportComponentFileRefField,
     ParallelComponentFileRefField,
@@ -29,7 +28,6 @@
 from ...exceptions import ValidationException
 from .._sweep.parameterized_sweep import ParameterizedSweepSchema
 from .._utils.data_binding_expression import support_data_binding_expression_for_fields
-from ..component.flow import FlowComponentSchema
 from ..core.fields import (
     ArmVersionedStr,
     ComputeField,
@@ -254,10 +252,6 @@ class ParallelSchema(BaseNodeSchema, ParameterizedParallelSchema):
                 # component file reference
                 ParallelComponentFileRefField(),
             ],
-            NodeType.FLOW_PARALLEL: [
-                NestedField(FlowComponentSchema, unknown=INCLUDE, dump_only=True),
-                ComponentYamlRefField(),
-            ],
         },
         plain_union_fields=[
             # for registry type assets

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/pipeline/component_job.py

@@ -381,18 +381,6 @@ class CommonYamlFields:
     """Schema."""
 
 
-class SchemaUrl:
-    """Schema urls.
-
-    Schema urls will be used in VSCode extension to validate yaml files. It will also be used to identify the
-    corresponding entity type of a yaml file, especially for some internal yaml files.
-    """
-
-    PROMPTFLOW_PREFIX = "https://azuremlschemas.azureedge.net/promptflow/"
-    PROMPTFLOW_FLOW = PROMPTFLOW_PREFIX + "latest/Flow.schema.json"
-    PROMPTFLOW_RUN = PROMPTFLOW_PREFIX + "latest/Run.schema.json"
-
-
 class GitProperties:
     """GitProperties is a class that defines the constants used by the SDK/CLI for Git operations.
 

sdk/ml/azure-ai-ml/azure/ai/ml/constants/_common.py

@@ -23,7 +23,6 @@ class NodeType(object):
     IMPORT = "import"
     SPARK = "spark"
     DATA_TRANSFER = "data_transfer"
-    FLOW_PARALLEL = "promptflow_parallel"
     # Note: container is not a real component type,
     # only used to mark component from container data.
     _CONTAINER = "_container"

sdk/ml/azure-ai-ml/azure/ai/ml/constants/_component.py

@@ -17,7 +17,6 @@
 from ...constants._common import ARM_ID_PREFIX
 from ...constants._component import NodeType
 from .._component.component import Component
-from .._component.flow import FlowComponent
 from .._component.parallel_component import ParallelComponent
 from .._inputs_outputs import Input, Output
 from .._job.job_resource_configuration import JobResourceConfiguration
@@ -123,28 +122,15 @@ def __init__(
         validate_attribute_type(attrs_to_check=locals(), attr_type_map=self._attr_type_map())
         kwargs.pop("type", None)
 
-        if isinstance(component, FlowComponent):
-            # make input definition fit actual inputs for flow component
-            with component._inputs._fit_inputs(inputs):  # pylint: disable=protected-access
-                BaseNode.__init__(
-                    self,
-                    type=NodeType.PARALLEL,
-                    component=component,
-                    inputs=inputs,
-                    outputs=outputs,
-                    compute=compute,
-                    **kwargs,
-                )
-        else:
-            BaseNode.__init__(
-                self,
-                type=NodeType.PARALLEL,
-                component=component,
-                inputs=inputs,
-                outputs=outputs,
-                compute=compute,
-                **kwargs,
-            )
+        BaseNode.__init__(
+            self,
+            type=NodeType.PARALLEL,
+            component=component,
+            inputs=inputs,
+            outputs=outputs,
+            compute=compute,
+            **kwargs,
+        )
         # init mark for _AttrDict
         self._init = True
 
@@ -323,7 +309,7 @@ def set_resources(
     @classmethod
     def _attr_type_map(cls) -> dict:
         return {
-            "component": (str, ParallelComponent, FlowComponent),
+            "component": (str, ParallelComponent),
             "retry_settings": (dict, RetrySettings),
             "resources": (dict, JobResourceConfiguration),
             "task": (dict, ParallelTask),