Skip to content

Commit aec8b7d

Browse files
lenchvolodymyrlenchvolodymyr
authored
Merge pull request #1 from lenchvolodymyr/feature/add-indexes
Feature/add indexes
2 parents c8bfe20 + 0a3df35 commit aec8b7d

File tree

10 files changed

+837
-308
lines changed

10 files changed

+837
-308
lines changed

‎adapter/0.1.5.json‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/**
2+
* Copyright © 2016-2020 by IntegrIT S.A. dba Hackolade. All rights reserved.
3+
*
4+
* The copyright to the computer software herein is the property of IntegrIT S.A.
5+
* The software may be used and/or copied only with the written permission of
6+
* IntegrIT S.A. or in accordance with the terms and conditions stipulated in
7+
* the agreement/contract under which the software has been supplied.
8+
*
9+
* {
10+
* "add": {
11+
* "entity": [<names of new property>],
12+
* "container": [<names of new property>],
13+
* "model": [<names of new property>],
14+
* "view": [<names of new property>],
15+
* "field": {
16+
* "<type>": [<names of new property>]
17+
* }
18+
* },
19+
* "delete": {
20+
* "entity": [<names of new property>],
21+
* "container": [<names of new property>],
22+
* "model": [<names of new property>],
23+
* "view": [<names of new property>],
24+
* "field": {
25+
* "<type>": [<names of new property>]
26+
* }
27+
* },
28+
* "modify": {
29+
* "entity": [
30+
* {
31+
* "from": { <properties that identify record> },
32+
* "to": { <properties that need to be changed> }
33+
* }
34+
* ],
35+
* "container": [],
36+
* "model": [],
37+
* "view": [],
38+
* "field": []
39+
* },
40+
* }
41+
*/
42+
{
43+
"modify": {
44+
"container": [
45+
["convertPathToFieldLink", ["partitionKey"]],
46+
["convertCosmosDbIndexes"]
47+
]
48+
}
49+
}

‎forward_engineering/api.js‎

Lines changed: 131 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,31 @@
11
const _ = require('lodash');
22
const applyToInstanceHelper = require('./applyToInstanceHelper');
3+
const getIndexPolicyScript = require('./getIndexPolicyScript');
4+
const scriptHelper = require('./scriptHelper');
35

46
const DEFAULT_INDENT = ' ';
57
let graphName = 'g';
68

79
module.exports = {
810
generateContainerScript(data, logger, cb) {
9-
let { collections, relationships, jsonData, containerData } = data;
11+
let { collections, relationships, jsonData, containerData, options } = data;
12+
const scriptId = _.get(options, 'targetScriptOptions.keyword');
1013
logger.clear();
1114
try {
15+
if (scriptId === 'cosmosdb') {
16+
return cb(
17+
null,
18+
JSON.stringify(
19+
{
20+
indexingPolicy: getIndexPolicyScript(_)(containerData),
21+
...scriptHelper.addItems(_)(containerData),
22+
},
23+
null,
24+
2
25+
)
26+
);
27+
}
28+
1229
let resultScript = '';
1330
const traversalSource = _.get(containerData, [0, 'traversalSource'], 'g');
1431
graphName = transformToValidGremlinName(traversalSource);
@@ -60,6 +77,7 @@ module.exports = {
6077
if (!data.containerData) {
6178
return cb({ message: 'Graph wasn\'t specified' });
6279
}
80+
const targetScriptOptions = data.targetScriptOptions || {};
6381
const containerProps = _.get(data.containerData, '[0]', {});
6482
if (!containerProps.dbId) {
6583
return cb({ message: 'Database id wasn\'t specified' });
@@ -68,46 +86,79 @@ module.exports = {
6886
if (!graphName) {
6987
return cb({ message: 'Graph name wasn\'t specified' });
7088
}
89+
const progress = createLogger(logger, containerProps.dbId, graphName);
7190

7291
const cosmosClient = applyToInstanceHelper.setUpDocumentClient(data);
92+
93+
progress('Create database if not exists ...');
94+
7395
await cosmosClient.databases.createIfNotExists({
7496
id: containerProps.dbId
7597
});
98+
99+
progress('Create container if not exists ...');
100+
76101
const containerResponse = await cosmosClient
77102
.database(containerProps.dbId)
78103
.containers.createIfNotExists({
79104
id: graphName,
80-
partitionKey: containerProps.partitionKey,
105+
partitionKey: getPartitionKey(_)(data.containerData),
81106
...(containerProps.autopilot
82107
? { maxThroughput: containerProps.throughput || 400 }
83108
: { throughput: containerProps.throughput || 400 }),
84109
defaultTtl: applyToInstanceHelper.getTTL(containerProps),
85110
});
86-
if (containerResponse.statusCode === 201) {
87-
const containerInstance = cosmosClient.database(containerProps.dbId).container(graphName);
88111

89-
const storedProcs = _.get(data.containerData, '[2].storedProcs', []);
112+
let functionsScripts;
113+
114+
if (targetScriptOptions.id === 'cosmosdb') {
115+
progress('Applying Cosmos DB script ...');
116+
const script = JSON.parse(data.script);
117+
118+
progress('Update indexing policy ...');
119+
120+
await containerResponse.container.replace({
121+
id: graphName,
122+
partitionKey: containerResponse.resource.partitionKey,
123+
indexingPolicy: updateIndexingPolicy(script.indexingPolicy),
124+
});
125+
126+
const storedProcs = _.get(script, 'Stored Procedures', []);
90127
if (storedProcs.length) {
91-
await applyToInstanceHelper.createStoredProcs(storedProcs, containerInstance);
128+
progress('Upload stored procs ...');
129+
await applyToInstanceHelper.createStoredProcs(storedProcs, containerResponse.container);
92130
}
93131

94-
const udfs = _.get(data.containerData, '[3].udfs', []);
132+
const udfs = _.get(script, 'User Defined Functions', []);
95133
if (udfs.length) {
96-
await applyToInstanceHelper.createUDFs(udfs, containerInstance);
134+
progress('Upload user defined functions ...');
135+
await applyToInstanceHelper.createUDFs(udfs, containerResponse.container);
97136
}
98-
const triggers = _.get(data.containerData, '[4].triggers', []);
137+
138+
const triggers = _.get(script, 'Triggers', []);
99139
if (triggers.length) {
100-
await applyToInstanceHelper.createTriggers(triggers, containerInstance);
140+
progress('Upload triggers ...');
141+
await applyToInstanceHelper.createTriggers(triggers, containerResponse.container);
101142
}
102-
}
103143

104-
const { labels, edges } = applyToInstanceHelper.parseScriptStatements(data.script);
105-
const gremlinClient = await applyToInstanceHelper.getGremlinClient(data, containerProps.dbId, graphName);
106-
await applyToInstanceHelper.runGremlinQueries(gremlinClient, labels);
107-
await applyToInstanceHelper.runGremlinQueries(gremlinClient, edges);
144+
} else {
145+
progress('Applying Gremlin script ...');
146+
147+
const { labels, edges } = applyToInstanceHelper.parseScriptStatements(data.script);
148+
const gremlinClient = await applyToInstanceHelper.getGremlinClient(data, containerProps.dbId, graphName);
108149

150+
progress('Uploading labels ...');
151+
152+
await applyToInstanceHelper.runGremlinQueries(gremlinClient, labels);
153+
154+
progress('Uploading edges ...');
155+
156+
await applyToInstanceHelper.runGremlinQueries(gremlinClient, edges);
157+
}
158+
109159
cb();
110160
} catch(err) {
161+
logger.log('error', mapError(err));
111162
cb(mapError(err));
112163
}
113164
},
@@ -126,6 +177,71 @@ module.exports = {
126177
}
127178
};
128179

180+
const getPartitionKey = (_) => (containerData) => {
181+
const partitionKey = _.get(containerData, '[0].partitionKey[0].name');
182+
183+
if (!partitionKey) {
184+
return;
185+
}
186+
187+
return '/' + partitionKey.split('.').slice(1).join('/');
188+
};
189+
190+
const updateIndexingPolicy = (indexes) => {
191+
const result = {...indexes};
192+
193+
if (Array.isArray(result.includedPaths)) {
194+
result.includedPaths = addDataType(result.includedPaths);
195+
}
196+
197+
if (Array.isArray(result.excludedPaths)) {
198+
result.excludedPaths = addDataType(result.excludedPaths);
199+
}
200+
201+
if (Array.isArray(result.spatialIndexes)) {
202+
result.spatialIndexes = result.spatialIndexes.map(addSpatialTypes);
203+
}
204+
205+
return result;
206+
};
207+
208+
const addDataType = (indexes) => {
209+
return indexes.map(index => {
210+
if (!Array.isArray(index.indexes)) {
211+
return index;
212+
}
213+
214+
return {
215+
...index,
216+
indexes: index.indexes.map(item => ({
217+
...item,
218+
dataType: item.dataType || 'String',
219+
})),
220+
};
221+
});
222+
};
223+
224+
const addSpatialTypes = (spatialIndex) => {
225+
if (Array.isArray(spatialIndex.types) && spatialIndex.types.length) {
226+
return spatialIndex;
227+
}
228+
229+
return {
230+
...spatialIndex,
231+
types: [
232+
"Point",
233+
"LineString",
234+
"Polygon",
235+
"MultiPolygon"
236+
]
237+
};
238+
};
239+
240+
const createLogger = (logger, containerName, entityName) => (message) => {
241+
logger.progress({ message, containerName, entityName });
242+
logger.log('info', { message }, 'Applying to instance');
243+
};
244+
129245
const generateVariables = variables => {
130246
return variables.reduce((script, variable) => {
131247
const key = variable.graphVariableKey;

‎forward_engineering/applyToInstanceHelper.js‎

Lines changed: 49 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
const _ = require('lodash');
2-
const { CosmosClient } = require('../reverse_engineering/node_modules/@azure/cosmos');
2+
const { CosmosClient, StoredProcedure, UserDefinedFunction, Trigger } = require('../reverse_engineering/node_modules/@azure/cosmos');
33
const gremlin = require('../reverse_engineering/node_modules/gremlin');
44

55

@@ -69,12 +69,6 @@ const applyToInstanceHelper = {
6969
});
7070
},
7171

72-
createStoredProcs(storedProcs, containerInstance) {
73-
return Promise.all(this.mapStoredProcs(storedProcs).map(proc => {
74-
return containerInstance.scripts.storedProcedures.create(proc);
75-
}));
76-
},
77-
7872
mapUDFs(udfs) {
7973
return udfs.map(udf => {
8074
return {
@@ -84,12 +78,6 @@ const applyToInstanceHelper = {
8478
});
8579
},
8680

87-
createUDFs(udfs, containerInstance) {
88-
return Promise.all(this.mapUDFs(udfs).map(udf => {
89-
return containerInstance.scripts.userDefinedFunctions.create(udf);
90-
}));
91-
},
92-
9381
mapTriggers(triggers) {
9482
return triggers.map(trigger => {
9583
return {
@@ -101,10 +89,55 @@ const applyToInstanceHelper = {
10189
});
10290
},
10391

92+
createStoredProcs(storedProcs, containerInstance) {
93+
return storedProcs.reduce(async (next, proc) => {
94+
await next;
95+
96+
try {
97+
return await containerInstance.scripts.storedProcedures.create(proc);
98+
} catch (error) {
99+
if (error.code !== 409) {
100+
throw error;
101+
}
102+
const result = new StoredProcedure(containerInstance, proc.id, containerInstance.clientContext);
103+
104+
return await result.replace(proc);
105+
}
106+
}, Promise.resolve());
107+
},
108+
109+
createUDFs(udfs, containerInstance) {
110+
return udfs.reduce(async (next, udf) => {
111+
await next;
112+
113+
try {
114+
return await containerInstance.scripts.userDefinedFunctions.create(udf);
115+
} catch (error) {
116+
if (error.code !== 409) {
117+
throw error;
118+
}
119+
const result = new UserDefinedFunction(containerInstance, udf.id, containerInstance.clientContext);
120+
121+
return await result.replace(udf);
122+
}
123+
}, Promise.resolve());
124+
},
125+
104126
createTriggers(triggers, containerInstance) {
105-
return Promise.all(this.mapTriggers(triggers).map(trigger => {
106-
return containerInstance.scripts.triggers.create(trigger);
107-
}));
127+
return triggers.reduce(async (next, trigger) => {
128+
await next;
129+
130+
try {
131+
return await containerInstance.scripts.triggers.create(trigger);
132+
} catch (error) {
133+
if (error.code !== 409) {
134+
throw error;
135+
}
136+
const result = new Trigger(containerInstance, trigger.id, containerInstance.clientContext);
137+
138+
return await result.replace(trigger);
139+
}
140+
}, Promise.resolve());
108141
},
109142

110143
getTTL(containerData) {

‎forward_engineering/config.json‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,13 @@
77
"level": {
88
"container": true,
99
"entity": false
10-
}
10+
},
11+
"options": [
12+
{ "name": "Gremlin Script", "keyword": "gremlin", "fileExtensions": [ {
13+
"label": "plain/text", "value": "gremlin"
14+
} ] },
15+
{ "name": "Cosmos DB Script", "keyword": "cosmosdb", "fileExtensions": [ {
16+
"label": "application/json", "value": "json"
17+
} ] }
18+
]
1119
}

0 commit comments

Comments
 (0)