Initial WorkspaceHub change for ML SDK (Azure#29842)

* template and base operation

contract temp change

Update schema and entities

Update template1

* update operation

* finished most operations

* template fixs

* Update Hub name to wprkspaceHub

update

update restclient

add kind support for 2023-02-01

update 2023-02-01-preview swagger and regenerate rest

update on parameter name, still need move change to febpreview and regenerate client

* remove lean workspace type and schema

remove lean operation

update template

small fix

update name and remove lean type

* revert workspace operation

Revert "revert workspace operation"

This reverts commit 10a74ab365de8356831f250130e2c5179ffd045e.

Revert "update name and remove lean type"

This reverts commit 598a1fd2f71416ede63a4c4b3e190cab34acf589.

Revert "Revert "update name and remove lean type""

This reverts commit f09898cdd542a0ac47940b10184111c83b090c2c.

Revert "Revert "revert workspace operation""

This reverts commit e654d146886b590b76468a83c39123b8a268c2a0.

revert change related to 02-01-preview

update to user 04-01-preview client

user 04-01-preview client for hub operation

fix lint error

lint error fix

 diagnose related package to 04-01-preview

* reformat code

update test client

small fix

fix ut

format fix

* prevent CMK update for lean workspace

* resolve rebase issue

Resolve merge conflict issue

* Update Hub related test code

* Update E2E test and configs

* Update model version for workspace

* rename workspacehub path

fix

* Update names

Update format

* Remove WorkspaceHub properties from workspace

* remove workspaceHub properties from workspace

remove workspacehub properties from workspace

* Fix import issue and tests

* Fix tests

* move dependent delete to workspacehub operation

* reformat code and fix feature store tests

fix

* Update YAMLRefDocLinks for workspaceHub

* format code

* rename workspace_hub and update creation flow

* fix format

fix

* associated_workspace and add support operation.

* Update template for project workspace

* reformat code

* fix format and code

* use different template for project workspace

* fix template bug

* Stop create LA workspace for workspacehub

* Update using new schema

* Update tests

Update tests

fix format

update format

Update more entity to 06-01 preview

fix tests

Fixed some tests

bug fixing

* Update docstring

* reflect comments

* Add missing changes

Author: ZhidaLiu

sdk/ml/azure-ai-ml/azure/ai/ml/__init__.py

@@ -32,6 +32,7 @@
     load_registry,
     load_workspace,
     load_workspace_connection,
+    load_workspace_hub,
 )
 
 module_logger = logging.getLogger(__name__)
@@ -65,6 +66,7 @@
     "load_workspace",
     "load_registry",
     "load_workspace_connection",
+    "load_workspace_hub",
     "load_model_package",
 ]
 

sdk/ml/azure-ai-ml/azure/ai/ml/_arm_deployments/arm_helper.py

@@ -21,6 +21,7 @@
     ArmConstants.UPDATE_ONLINE_ENDPOINT_TYPE: "update_online_endpoint.json",
     ArmConstants.WORKSPACE_BASE: "workspace_base.json",
     ArmConstants.WORKSPACE_PARAM: "workspace_param.json",
+    ArmConstants.WORKSPACE_PROJECT: "workspace_project.json",
 }
 
 

sdk/ml/azure-ai-ml/azure/ai/ml/_arm_deployments/arm_templates/workspace_base.json

@@ -486,6 +486,27 @@
             "metadata": {
                 "description": "A flag to determine if workspace has data isolation enabled. The flag can only be set at the creation phase, it can't be updated."
             }
+        },
+        "workspace_hub_config" : {
+            "type": "object",
+            "defaultValue": {},
+            "metadata": {
+                "description": "Additional configuration for WorkspaceHub."
+            }
+        },
+        "existing_workspaces" : {
+            "type": "array",
+            "defaultValue": [],
+            "metadata": {
+                "description": "A list of existing workspaces used by Hub to perform convert."
+            }
+        },
+        "workspace_hub" : {
+            "type": "string",
+            "defaultValue": "",
+            "metadata": {
+                "description": "Resource Id of Hub used for lean workspace."
+            }
         }
     },
     "variables": {
@@ -518,7 +539,8 @@
         "defaultPEConnections": "[array(variables('privateEndpointSettings'))]",
         "privateEndpointDeploymentName": "[concat('DeployPrivateEndpoint-', uniqueString(parameters('privateEndpointName')))]",
         "offlineStoreConnectionName": "[if(equals(parameters('offline_store_connection_name'), ''), 'OfflineStoreConnectionName', parameters('offline_store_connection_name'))]",
-        "onlineStoreConnectionName": "[if(equals(parameters('online_store_connection_name'), ''), 'OnlineStoreConnectionName', parameters('online_store_connection_name'))]"
+        "onlineStoreConnectionName": "[if(equals(parameters('online_store_connection_name'), ''), 'OnlineStoreConnectionName', parameters('online_store_connection_name'))]",
+        "isWorkspaceHub": "[equals(parameters('kind'), 'hub')]"
     },
     "resources": [
         {
@@ -631,7 +653,7 @@
             }
         },
         {
-            "condition": "[and(variables('enablePE'), equals(parameters('applicationInsightsOption'), 'new'))]",
+            "condition": "[and(variables('enablePE'), equals(parameters('applicationInsightsOption'), 'new'), not(variables('isWorkspaceHub')))]",
             "type": "Microsoft.OperationalInsights/workspaces",
             "tags": "[parameters('tagValues')]",
             "apiVersion": "2020-08-01",
@@ -643,7 +665,7 @@
             }
         },
         {
-            "condition": "[and(variables('enablePE'), equals(parameters('applicationInsightsOption'), 'new'))]",
+            "condition": "[and(variables('enablePE'), equals(parameters('applicationInsightsOption'), 'new'), not(variables('isWorkspaceHub')))]",
             "type": "Microsoft.Insights/components",
             "tags": "[parameters('tagValues')]",
             "apiVersion": "2020-02-02-preview",
@@ -678,7 +700,7 @@
                 "description": "[parameters('description')]",
                 "storageAccount": "[variables('storageAccount')]",
                 "keyVault": "[variables('keyVault')]",
-                "applicationInsights": "[variables('applicationInsights')]",
+                "applicationInsights": "[if(variables('isWorkspaceHub'), json('null'), variables('applicationInsights'))]",
                 "containerRegistry": "[if(not(equals(parameters('containerRegistryOption'), 'none')), variables('containerRegistry'), json('null'))]",
                 "hbiWorkspace": "[parameters('confidential_data')]",
                 "imageBuildCompute": "[parameters('imageBuildCompute')]",
@@ -704,7 +726,9 @@
                     "offlinestoreconnectionname": "[parameters('offline_store_connection_name')]",
                     "onlinestoreconnectionname": "[parameters('online_store_connection_name')]"
                 },
-                "enableDataIsolation": "[parameters('enable_data_isolation')]"
+                "enableDataIsolation": "[parameters('enable_data_isolation')]",
+                "workspaceHubConfig": "[parameters('workspace_hub_config')]",
+                "existingWorkspaces": "[parameters('existing_workspaces')]"
             }
         },
         {

sdk/ml/azure-ai-ml/azure/ai/ml/_arm_deployments/arm_templates/workspace_param.json

@@ -152,6 +152,15 @@
     "online_store_connection_target" : {
         "value": ""
     },
+    "workspace_hub_config" : {
+        "value": {}
+    },
+    "existing_workspaces" : {
+        "value": []
+    },
+    "workspace_hub" : {
+        "value": ""
+    },
     "enable_data_isolation": {
         "value": "false"
     }