Skip to content

Commit a7e7a5b

Browse files
davidkhaladavidkhala
committed
try patch
microsoft/Purview-ADB-Lineage-Solution-Accelerator#235
1 parent 514020d commit a7e7a5b

File tree

2 files changed

+99
-16
lines changed

2 files changed

+99
-16
lines changed

‎purview/lineage/palsa/Custom_Types.json‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
{
2+
"entityDefs": [
3+
{
4+
"category": "ENTITY",
5+
"name": "purview_custom_connector_generic_column",
6+
"description": "purview_custom_connector_generic_column",
7+
"serviceType": "Purview Custom Connector",
8+
"options": {
9+
"schemaAttributes": "[\"data_type\"]"
10+
},
11+
"attributeDefs": [
12+
{
13+
"name": "data_type",
14+
"typeName": "string",
15+
"isOptional": false,
16+
"cardinality": "SINGLE",
17+
"valuesMinCount": 1,
18+
"valuesMaxCount": 1,
19+
"isUnique": false,
20+
"isIndexable": false,
21+
"includeInNotification": false
22+
}
23+
],
24+
"superTypes": [
25+
"DataSet"
26+
],
27+
"subTypes": [],
28+
"relationshipAttributeDefs": []
29+
},
30+
{
31+
"category": "ENTITY",
32+
"name": "purview_custom_connector_generic_entity_with_columns",
33+
"description": "purview_custom_connector_generic_entity_with_columns",
34+
"serviceType": "Purview Custom Connector",
35+
"options": {
36+
"schemaElementsAttribute": "columns"
37+
},
38+
"attributeDefs": [
39+
{
40+
"name": "purview_qualifiedName",
41+
"typeName": "string",
42+
"isOptional": true,
43+
"cardinality": "SINGLE",
44+
"valuesMinCount": 0,
45+
"valuesMaxCount": 1,
46+
"isUnique": false,
47+
"isIndexable": false,
48+
"includeInNotification": false
49+
},
50+
{
51+
"name": "original_source",
52+
"typeName": "string",
53+
"isOptional": true,
54+
"cardinality": "SINGLE",
55+
"valuesMinCount": 0,
56+
"valuesMaxCount": 1,
57+
"isUnique": false,
58+
"isIndexable": false,
59+
"includeInNotification": false
60+
}
61+
],
62+
"superTypes": [
63+
"DataSet"
64+
],
65+
"subTypes": [],
66+
"relationshipAttributeDefs": []
67+
}
68+
]
69+
}

‎purview/lineage/palsa/standalone.sh‎

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,26 @@ deploy-connector() {
1616
curl https://raw.githubusercontent.com/microsoft/Purview-ADB-Lineage-Solution-Accelerator/refs/heads/release/2.3/deployment/infra/newdeploymenttemp.json -O
1717

1818
az deployment group create --resource-group $rg --name $deploymentName --template-file "./newdeploymenttemp.json" --parameters purviewName=$purviewName prefixName= clientid=$clientid clientsecret=$clientsecret resourceTagValues={} --output none
19-
# if failure, clean up by `az deployment group delete --name newdeploymenttemp --resource-group $rg`
19+
2020
rm newdeploymenttemp.json
21-
21+
2222
}
2323

2424
deploy-stats() {
25-
az deployment group show --name $deploymentName --resource-group $rg
25+
az deployment group show --name $deploymentName --resource-group $rg $@
26+
# if failure, clean up by `az deployment group delete --name newdeploymenttemp --resource-group $rg`
27+
}
28+
deploy-env() {
29+
deploy-stats --query properties.outputs >stats.value.json
30+
export FUNNAME=$(jq -r '.functionAppName.value' stats.value.json)
31+
export KVNAME=$(jq -r '.kvName.value' stats.value.json)
32+
export ADLSNAME=$(jq -r '.storageAccountName.value' stats.value.json) # storageAccountName
33+
34+
# You can see there are 2 keys created in same time. We pick the second one here
35+
adls_key=$(az storage account keys list -g $rg -n $ADLSNAME --query '[1].value' --output tsv)
36+
echo $adls_key
37+
export RGLOCATION=$purviewlocation # TODO clean this
38+
rm stats.value.json
2639
}
2740

2841
# Install necessary types into your Purview instance
@@ -34,23 +47,24 @@ config-purview() {
3447
local login_endpoint="https://login.microsoftonline.com/$tenantid/oauth2/token"
3548
acc_purview_token=$(curl $login_endpoint --data "resource=https://purview.azure.net&client_id=$clientid&client_secret=$clientsecret&grant_type=client_credentials" -H Metadata:true -s | jq -r '.access_token')
3649

37-
curl https://raw.githubusercontent.com/microsoft/Purview-ADB-Lineage-Solution-Accelerator/refs/heads/release/2.3/deployment/infra/Custom_Types.json -O
50+
# learnt from https://github.com/microsoft/Purview-ADB-Lineage-Solution-Accelerator/pull/235
51+
curl https://raw.githubusercontent.com/davidkhala/Microsoft/refs/heads/main/purview/lineage/palsa/Custom_Types.json -O
3852

3953
curl -s -X POST $purview_endpoint/catalog/api/atlas/v2/types/typedefs -H "Authorization: Bearer $acc_purview_token" -H "Content-Type: application/json" -d @Custom_Types.json
40-
# TODO troubleshoot error
41-
# {"requestId":"d1fb3a39-9016-44c0-9a7a-8164419d4188","errorCode":"ATLAS-400-00-01A","errorMessage":"invalid parameters: invalid payload, expect schemaAttributes in purview_custom_connector_generic_column should be list of string, but found: data_type"}
54+
4255
rm Custom_Types.json
4356
}
57+
config-databricks() {
58+
# TODO databricks part
59+
if ! unzip -v; then
60+
echo "unzip is required. Please find and install on your OS"
61+
exit 1
62+
fi
4463

45-
# TODO databricks part
46-
if ! unzip -v; then
47-
echo "unzip is required. Please find and install on your OS"
48-
exit 1
49-
fi
64+
if ! databricks -v; then
65+
# install DataBricks CLI
66+
curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sudo sh
67+
fi
68+
}
5069

51-
if ! databricks -v; then
52-
# install DataBricks CLI
53-
curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sudo sh
54-
fi
55-
# TODO databricks part
5670
$@

0 commit comments

Comments
 (0)