From ec0e5f3f9491aec141fd716a34068382c549186d Mon Sep 17 00:00:00 2001 From: RenzoPrats Date: Tue, 16 Jan 2024 12:21:08 -0300 Subject: [PATCH 1/5] feat: removed pulljob --- __tests__/models/pullJob.test.ts | 65 -- __tests__/schema/query/pullJob.test.ts | 45 -- migrations/1664441608216-placeholders.ts | 32 +- src/index.ts | 2 - src/models/index.ts | 1 - src/models/pullJob.model.ts | 63 -- src/routes/upload/index.ts | 26 - src/schema/mutation/addPullJob.mutation.ts | 124 ---- src/schema/mutation/deletePullJob.mutation.ts | 52 -- src/schema/mutation/editPullJob.mutation.ts | 128 ---- src/schema/mutation/index.ts | 6 - src/schema/query/index.ts | 2 - src/schema/query/pullJobs.query.ts | 84 --- src/schema/types/application.type.ts | 13 - src/schema/types/index.ts | 1 - src/schema/types/pullJob.type.ts | 66 -- src/security/defineUserAbility.ts | 6 +- src/server/EIOSOwnernshipMapping.ts | 65 -- src/server/pullJobScheduler.ts | 670 ------------------ 19 files changed, 3 insertions(+), 1448 deletions(-) delete mode 100644 __tests__/models/pullJob.test.ts delete mode 100644 __tests__/schema/query/pullJob.test.ts delete mode 100644 src/models/pullJob.model.ts delete mode 100644 src/schema/mutation/addPullJob.mutation.ts delete mode 100644 src/schema/mutation/deletePullJob.mutation.ts delete mode 100644 src/schema/mutation/editPullJob.mutation.ts delete mode 100644 src/schema/query/pullJobs.query.ts delete mode 100644 src/schema/types/pullJob.type.ts delete mode 100644 src/server/EIOSOwnernshipMapping.ts delete mode 100644 src/server/pullJobScheduler.ts diff --git a/__tests__/models/pullJob.test.ts b/__tests__/models/pullJob.test.ts deleted file mode 100644 index 928f6e8e5..000000000 --- a/__tests__/models/pullJob.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { PullJob, Form, Channel, ApiConfiguration, Resource } from '@models'; -import { status } from '@const/enumTypes'; -import { faker } from '@faker-js/faker'; - -/** - * Test PullJob Model. - */ -describe('PullJob models tests', () => { - let pullJob: PullJob; - test('test PullJob model with correct data', async () => { - const apiConfiguration = await new ApiConfiguration({ - name: faker.internet.userName(), - }).save(); - const formName = faker.random.alpha(10); - - const resource = await new Resource({ - name: formName, - }).save(); - - const form = await new Form({ - name: formName, - graphQLTypeName: formName, - resource: resource._id, - }).save(); - const channel = await new Channel({ - title: faker.internet.userName(), - }).save(); - - for (let i = 0; i < 1; i++) { - const inputData = { - name: faker.word.adjective(), - status: status.active, - apiConfiguration: apiConfiguration._id, - url: faker.internet.url(), - path: faker.system.directoryPath(), - schedule: '* * * * *', - convertTo: form._id, - channel: channel._id, - }; - pullJob = await new PullJob(inputData).save(); - expect(pullJob._id).toBeDefined(); - } - }); - - test('test pullJob with duplicate name', async () => { - const duplicatePullJob = { - name: pullJob.name, - }; - expect(async () => - new PullJob(duplicatePullJob).save() - ).rejects.toThrowError( - 'E11000 duplicate key error collection: test.pulljobs index: name_1 dup key' - ); - }); - - test('test PullJob model with wrong status', async () => { - for (let i = 0; i < 1; i++) { - const inputData = { - name: faker.word.adjective(), - status: faker.word.adjective(), - }; - expect(async () => new PullJob(inputData).save()).rejects.toThrow(Error); - } - }); -}); diff --git a/__tests__/schema/query/pullJob.test.ts b/__tests__/schema/query/pullJob.test.ts deleted file mode 100644 index 6df36c6c9..000000000 --- a/__tests__/schema/query/pullJob.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { ApolloServer } from 'apollo-server-express'; -import schema from '../../../src/schema'; -import { SafeTestServer } from '../../server.setup'; -import { PullJob, Role } from '@models'; - -let server: ApolloServer; - -/** - * Test PullJob query. - */ -describe('PullJob query tests', () => { - const query = '{ pullJobs { totalCount, edges { node { id } } } }'; - - test('query with wrong user returns error', async () => { - server = await SafeTestServer.createApolloTestServer(schema, { - name: 'Wrong user', - roles: [], - }); - const result = await server.executeOperation({ query }); - expect(result.errors).toBeUndefined(); - expect(result).toHaveProperty(['data', 'pullJobs', 'totalCount']); - expect(result.data?.pullJobs.edges).toEqual([]); - expect(result.data?.pullJobs.totalCount).toEqual(0); - }); - - test('query with admin user returns expected number of pullJobs', async () => { - const count = await PullJob.countDocuments(); - const admin = await Role.findOne( - { title: 'admin' }, - 'id permissions' - ).populate({ - path: 'permissions', - model: 'Permission', - }); - server = await SafeTestServer.createApolloTestServer(schema, { - name: 'Admin user', - roles: [admin], - }); - const result = await server.executeOperation({ query }); - - expect(result.errors).toBeUndefined(); - expect(result).toHaveProperty(['data', 'pullJobs', 'totalCount']); - expect(result.data?.pullJobs.totalCount).toEqual(count); - }); -}); diff --git a/migrations/1664441608216-placeholders.ts b/migrations/1664441608216-placeholders.ts index d211ef075..a43aea590 100644 --- a/migrations/1664441608216-placeholders.ts +++ b/migrations/1664441608216-placeholders.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-loop-func */ import { startDatabaseForMigration } from '../src/utils/migrations/database.helper'; -import { Dashboard, Resource, PullJob, ReferenceData } from '../src/models'; +import { Dashboard, Resource, ReferenceData } from '../src/models'; import { Placeholder } from '../src/const/placeholders'; import { logger } from '../src/services/logger.service'; import get from 'lodash/get'; @@ -174,36 +174,6 @@ const migratePlaceholders = async () => { } await Promise.all(layoutPromises); - // === PULLJOB === - logger.info('Start pulljob update'); - const pullJobs = await PullJob.find({}); - const mappingPromises: Promise[] = []; - for (const pullJob of pullJobs) { - let modified = false; - for (const key in pullJob.mapping) { - if (pullJob.mapping[key].startsWith('$$')) { - pullJob.mapping[key] = '{{' + pullJob.mapping[key].slice(2) + '}}'; - modified = true; - } - } - if (modified) { - pullJob.markModified('mapping'); - mappingPromises.push( - pullJob.save().then( - () => { - logger.info('Pulljob "' + pullJob.name + '" mapping updated.'); - }, - (err) => { - logger.info( - 'Could not update pullJob "' + pullJob.name + '" mapping.\n' + err - ); - } - ) - ); - } - } - await Promise.all(mappingPromises); - // === REFERENCE DATA === logger.info('Start reference data update'); const referenceDatas = await ReferenceData.find({ diff --git a/src/index.ts b/src/index.ts index e1c3cd729..81dd6ed28 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,5 @@ import { SafeServer } from './server'; import mongoose from 'mongoose'; -import pullJobScheduler from './server/pullJobScheduler'; import customNotificationScheduler from './server/customNotificationScheduler'; import { startDatabase } from './server/database'; import config from 'config'; @@ -31,7 +30,6 @@ startDatabase(); mongoose.connection.once('open', () => { logger.log({ level: 'info', message: '📶 Connected to database' }); // subscriberSafe(); - pullJobScheduler(); customNotificationScheduler(); }); diff --git a/src/models/index.ts b/src/models/index.ts index 191acf18a..b90203b86 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -10,7 +10,6 @@ export * from './page.model'; export * from './permission.model'; export * from './positionAttribute.model'; export * from './positionAttributeCategory.model'; -export * from './pullJob.model'; export * from './record.model'; export * from './referenceData.model'; export * from './resource.model'; diff --git a/src/models/pullJob.model.ts b/src/models/pullJob.model.ts deleted file mode 100644 index daac92649..000000000 --- a/src/models/pullJob.model.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { AccessibleRecordModel, accessibleRecordsPlugin } from '@casl/mongoose'; -import mongoose, { Schema, Document } from 'mongoose'; -import { status } from '@const/enumTypes'; -import { ApiConfiguration } from './apiConfiguration.model'; -import * as cron from 'cron-validator'; - -/** Mongoose pull job schema declaration */ -const pullJobSchema = new Schema({ - name: String, - status: { - type: String, - enum: Object.values(status), - }, - apiConfiguration: { - type: mongoose.Schema.Types.ObjectId, - ref: 'ApiConfiguration', - }, - url: String, - path: String, - schedule: { - type: String, - validate: { - validator: function (value) { - return value ? cron.isValidCron(value) : false; - }, - }, - }, - convertTo: { - type: mongoose.Schema.Types.ObjectId, - ref: 'Form', - }, - mapping: mongoose.Schema.Types.Mixed, - uniqueIdentifiers: [String], - channel: { - type: mongoose.Schema.Types.ObjectId, - ref: 'Channel', - }, -}); - -pullJobSchema.index({ name: 1 }, { unique: true }); - -/** Pull job documents interface declaration */ -export interface PullJob extends Document { - kind: 'PullJob'; - name: string; - status: string; - apiConfiguration: ApiConfiguration; - url: string; - path: string; - schedule: string; - convertTo: string; - mapping: any; - uniqueIdentifiers: string[]; - channel: string; -} -pullJobSchema.plugin(accessibleRecordsPlugin); - -/** Mongoose pull job model definition */ -// eslint-disable-next-line @typescript-eslint/no-redeclare -export const PullJob = mongoose.model>( - 'PullJob', - pullJobSchema -); diff --git a/src/routes/upload/index.ts b/src/routes/upload/index.ts index 63beb674c..5dad56780 100644 --- a/src/routes/upload/index.ts +++ b/src/routes/upload/index.ts @@ -17,7 +17,6 @@ import i18next from 'i18next'; import get from 'lodash/get'; import { logger } from '@services/logger.service'; import { accessibleBy } from '@casl/mongoose'; -import { insertRecords as insertRecordsPulljob } from '@server/pullJobScheduler'; import jwtDecode from 'jwt-decode'; /** File size limit, in bytes */ @@ -213,31 +212,6 @@ router.post('/resource/records/:id', async (req: any, res) => { } }); -/** - * Upload a list of records for a resource in json format - */ -router.post('/resource/insert', async (req: any, res) => { - try { - const authToken = req.headers.authorization.split(' ')[1]; - const decodedToken = jwtDecode(authToken) as any; - - // Block if connected with user to Service - if (!decodedToken.email && !decodedToken.name) { - const insertRecordsMessage = await insertRecordsPulljob( - req.body.records, - req.body.parameters, - true, - true - ); - return res.status(200).send(insertRecordsMessage); - } - return res.status(400).send(req.t('common.errors.permissionNotGranted')); - } catch (err) { - logger.error(err.message, { stack: err.stack }); - return res.status(500).send(req.t('common.errors.internalServerError')); - } -}); - /** * Import a list of users for an application from an uploaded xlsx file */ diff --git a/src/schema/mutation/addPullJob.mutation.ts b/src/schema/mutation/addPullJob.mutation.ts deleted file mode 100644 index c36411c2a..000000000 --- a/src/schema/mutation/addPullJob.mutation.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { - GraphQLError, - GraphQLID, - GraphQLList, - GraphQLNonNull, - GraphQLString, -} from 'graphql'; -import { PullJobType } from '../types'; -import { StatusType, status } from '@const/enumTypes'; -import { Channel, Form, PullJob } from '@models'; -import { StatusEnumType } from '@const/enumTypes'; -import GraphQLJSON from 'graphql-type-json'; -import { scheduleJob, unscheduleJob } from '../../server/pullJobScheduler'; -import { AppAbility } from '@security/defineUserAbility'; -import { logger } from '@services/logger.service'; -import { graphQLAuthCheck } from '@schema/shared'; -import { Types } from 'mongoose'; -import { Context } from '@server/apollo/context'; - -/** Arguments for the addPullJob mutation */ -type AddPullJobArgs = { - name: string; - status: StatusType; - apiConfiguration: string | Types.ObjectId; - url?: string; - path?: string; - schedule?: string; - convertTo?: string | Types.ObjectId; - mapping?: any; - uniqueIdentifiers?: string[]; - channel?: string | Types.ObjectId; -}; - -/** - * Creates a new pulljob. - * Throw an error if the user is not logged or authorized or if the form or channel aren't found. - */ -export default { - type: PullJobType, - args: { - name: { type: new GraphQLNonNull(GraphQLString) }, - status: { type: new GraphQLNonNull(StatusEnumType) }, - apiConfiguration: { type: new GraphQLNonNull(GraphQLID) }, - url: { type: GraphQLString }, - path: { type: GraphQLString }, - schedule: { type: GraphQLString }, - convertTo: { type: GraphQLID }, - mapping: { type: GraphQLJSON }, - uniqueIdentifiers: { type: new GraphQLList(GraphQLString) }, - channel: { type: GraphQLID }, - }, - async resolve(parent, args: AddPullJobArgs, context: Context) { - graphQLAuthCheck(context); - try { - const user = context.user; - - const ability: AppAbility = user.ability; - if (ability.can('create', 'PullJob')) { - if (args.convertTo) { - const form = await Form.findById(args.convertTo); - if (!form) - throw new GraphQLError( - context.i18next.t('common.errors.dataNotFound') - ); - } - - if (args.channel) { - const filters = { - _id: args.channel, - }; - const channel = await Channel.findOne(filters); - if (!channel) - throw new GraphQLError( - context.i18next.t('common.errors.dataNotFound') - ); - } - - try { - // Create a new PullJob - const pullJob = new PullJob({ - name: args.name, - status: args.status, - apiConfiguration: args.apiConfiguration, - url: args.url, - path: args.path, - schedule: args.schedule, - convertTo: args.convertTo, - mapping: args.mapping, - uniqueIdentifiers: args.uniqueIdentifiers, - channel: args.channel, - }); - await pullJob.save(); - - // If the pullJob is active, schedule it immediately - if (args.status === status.active) { - const fullPullJob = await PullJob.findById(pullJob.id).populate({ - path: 'apiConfiguration', - model: 'ApiConfiguration', - }); - scheduleJob(fullPullJob); - } else { - unscheduleJob(pullJob); - } - return pullJob; - } catch (err) { - logger.error(err.message); - throw new GraphQLError(err.message); - } - } else { - throw new GraphQLError( - context.i18next.t('common.errors.permissionNotGranted') - ); - } - } catch (err) { - logger.error(err.message, { stack: err.stack }); - if (err instanceof GraphQLError) { - throw new GraphQLError(err.message); - } - throw new GraphQLError( - context.i18next.t('common.errors.internalServerError') - ); - } - }, -}; diff --git a/src/schema/mutation/deletePullJob.mutation.ts b/src/schema/mutation/deletePullJob.mutation.ts deleted file mode 100644 index dce9f9190..000000000 --- a/src/schema/mutation/deletePullJob.mutation.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { GraphQLError, GraphQLID, GraphQLNonNull } from 'graphql'; -import { PullJobType } from '../types'; -import { PullJob } from '@models'; -import { AppAbility } from '@security/defineUserAbility'; -import { unscheduleJob } from '../../server/pullJobScheduler'; -import { logger } from '@services/logger.service'; -import { accessibleBy } from '@casl/mongoose'; -import { graphQLAuthCheck } from '@schema/shared'; -import { Types } from 'mongoose'; -import { Context } from '@server/apollo/context'; - -/** Arguments for the deletePullJob mutation */ -type DeletePullJobArgs = { - id: string | Types.ObjectId; -}; - -/** - * Delete a pullJob - */ -export default { - type: PullJobType, - args: { - id: { type: new GraphQLNonNull(GraphQLID) }, - }, - async resolve(parent, args: DeletePullJobArgs, context: Context) { - graphQLAuthCheck(context); - try { - const user = context.user; - const ability: AppAbility = user.ability; - - const filters = PullJob.find(accessibleBy(ability, 'delete').PullJob) - .where({ _id: args.id }) - .getFilter(); - const pullJob = await PullJob.findOneAndDelete(filters); - if (!pullJob) - throw new GraphQLError( - context.i18next.t('common.errors.permissionNotGranted') - ); - - unscheduleJob(pullJob); - return pullJob; - } catch (err) { - logger.error(err.message, { stack: err.stack }); - if (err instanceof GraphQLError) { - throw new GraphQLError(err.message); - } - throw new GraphQLError( - context.i18next.t('common.errors.internalServerError') - ); - } - }, -}; diff --git a/src/schema/mutation/editPullJob.mutation.ts b/src/schema/mutation/editPullJob.mutation.ts deleted file mode 100644 index 4cf2a6d0c..000000000 --- a/src/schema/mutation/editPullJob.mutation.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { - GraphQLError, - GraphQLID, - GraphQLList, - GraphQLNonNull, - GraphQLString, -} from 'graphql'; -import { PullJobType } from '../types'; -import { StatusType, status } from '@const/enumTypes'; -import { Channel, Form, PullJob } from '@models'; -import { AppAbility } from '@security/defineUserAbility'; -import { StatusEnumType } from '@const/enumTypes'; -import GraphQLJSON from 'graphql-type-json'; -import { scheduleJob, unscheduleJob } from '../../server/pullJobScheduler'; -import { logger } from '@services/logger.service'; -import { accessibleBy } from '@casl/mongoose'; -import { graphQLAuthCheck } from '@schema/shared'; -import { Types } from 'mongoose'; -import { Context } from '@server/apollo/context'; - -/** Arguments for the editPullJob mutation */ -type EditPullJobArgs = { - id: string | Types.ObjectId; - name?: string; - status?: StatusType; - apiConfiguration?: string | Types.ObjectId; - url?: string; - path?: string; - schedule?: string; - convertTo?: string | Types.ObjectId; - mapping?: any; - uniqueIdentifiers?: string[]; - channel?: string | Types.ObjectId; -}; - -/** - * Edit an existing pullJob if authorized. - */ -export default { - type: PullJobType, - args: { - id: { type: new GraphQLNonNull(GraphQLID) }, - name: { type: GraphQLString }, - status: { type: StatusEnumType }, - apiConfiguration: { type: GraphQLID }, - url: { type: GraphQLString }, - path: { type: GraphQLString }, - schedule: { type: GraphQLString }, - convertTo: { type: GraphQLID }, - mapping: { type: GraphQLJSON }, - uniqueIdentifiers: { type: new GraphQLList(GraphQLString) }, - channel: { type: GraphQLID }, - }, - async resolve(parent, args: EditPullJobArgs, context: Context) { - graphQLAuthCheck(context); - try { - const user = context.user; - const ability: AppAbility = user.ability; - - if (args.convertTo) { - const form = await Form.findById(args.convertTo); - if (!form) - throw new GraphQLError( - context.i18next.t('common.errors.dataNotFound') - ); - } - - if (args.channel) { - const filters = { - _id: args.channel, - }; - const channel = await Channel.findOne(filters); - if (!channel) - throw new GraphQLError( - context.i18next.t('common.errors.dataNotFound') - ); - } - - const update = {}; - Object.assign( - update, - args.name && { name: args.name }, - args.status && { status: args.status }, - args.apiConfiguration && { apiConfiguration: args.apiConfiguration }, - args.url && { url: args.url }, - args.path && { path: args.path }, - args.schedule && { schedule: args.schedule }, - args.convertTo && { convertTo: args.convertTo }, - args.mapping && { mapping: args.mapping }, - args.uniqueIdentifiers && { uniqueIdentifiers: args.uniqueIdentifiers }, - args.channel && { channel: args.channel } - ); - const filters = PullJob.find(accessibleBy(ability, 'update').PullJob) - .where({ _id: args.id }) - .getFilter(); - try { - const pullJob = await PullJob.findOneAndUpdate(filters, update, { - new: true, - runValidators: true, - }).populate({ - path: 'apiConfiguration', - model: 'ApiConfiguration', - }); - if (!pullJob) - throw new GraphQLError( - context.i18next.t('common.errors.dataNotFound') - ); - if (pullJob.status === status.active) { - scheduleJob(pullJob); - } else { - unscheduleJob(pullJob); - } - return pullJob; - } catch (err) { - logger.error(err.message); - throw new GraphQLError(err.message); - } - } catch (err) { - logger.error(err.message, { stack: err.stack }); - if (err instanceof GraphQLError) { - throw new GraphQLError(err.message); - } - throw new GraphQLError( - context.i18next.t('common.errors.internalServerError') - ); - } - }, -}; diff --git a/src/schema/mutation/index.ts b/src/schema/mutation/index.ts index 5814f2446..33602be09 100644 --- a/src/schema/mutation/index.ts +++ b/src/schema/mutation/index.ts @@ -52,9 +52,6 @@ import editUserProfile from './editUserProfile.mutation'; import addApiConfiguration from './addApiConfiguration.mutation'; import editApiConfiguration from './editApiConfiguration.mutation'; import deleteApiConfiguration from './deleteApiConfiguration.mutation'; -import addPullJob from './addPullJob.mutation'; -import editPullJob from './editPullJob.mutation'; -import deletePullJob from './deletePullJob.mutation'; import toggleApplicationLock from './toggleApplicationLock.mutation'; import addUsers from './addUsers.mutation'; import addLayout from './addLayout.mutation'; @@ -104,7 +101,6 @@ const Mutation = new GraphQLObjectType({ addPage, addPositionAttribute, addPositionAttributeCategory, - addPullJob, addRecord, addReferenceData, addRole, @@ -124,7 +120,6 @@ const Mutation = new GraphQLObjectType({ deleteLayout, deletePage, deletePositionAttributeCategory, - deletePullJob, deleteRecord, deleteRecords, deleteReferenceData, @@ -147,7 +142,6 @@ const Mutation = new GraphQLObjectType({ editPage, editPageContext, editPositionAttributeCategory, - editPullJob, editRecord, editRecords, editReferenceData, diff --git a/src/schema/query/index.ts b/src/schema/query/index.ts index 74920da62..bff4a4246 100644 --- a/src/schema/query/index.ts +++ b/src/schema/query/index.ts @@ -27,7 +27,6 @@ import channels from './channels.query'; import positionAttributes from './positionAttributes.query'; import apiConfiguration from './apiConfiguration.query'; import apiConfigurations from './apiConfigurations.query'; -import pullJobs from './pullJobs.query'; import referenceData from './referenceData.query'; import referenceDatas from './referenceDatas.query'; import recordHistory from './recordHistory.query'; @@ -60,7 +59,6 @@ const Query = new GraphQLObjectType({ page, pages, permissions, - pullJobs, record, records, recordsAggregation, diff --git a/src/schema/query/pullJobs.query.ts b/src/schema/query/pullJobs.query.ts deleted file mode 100644 index e6050b4e4..000000000 --- a/src/schema/query/pullJobs.query.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { GraphQLError, GraphQLID, GraphQLInt } from 'graphql'; -import { PullJobConnectionType, encodeCursor, decodeCursor } from '../types'; -import { PullJob } from '@models'; -import { AppAbility } from '@security/defineUserAbility'; -import { logger } from '@services/logger.service'; -import checkPageSize from '@utils/schema/errors/checkPageSize.util'; -import { accessibleBy } from '@casl/mongoose'; -import { graphQLAuthCheck } from '@schema/shared'; -import { Context } from '@server/apollo/context'; - -/** Default page size */ -const DEFAULT_FIRST = 10; - -/** Arguments for the pullJobs query */ -type PullJobsArgs = { - first?: number; - afterCursor?: string; -}; - -/** - * Return all pull jobs available for the logged user. - * Throw GraphQL error if not logged. - */ -export default { - type: PullJobConnectionType, - args: { - first: { type: GraphQLInt }, - afterCursor: { type: GraphQLID }, - }, - async resolve(parent, args: PullJobsArgs, context: Context) { - graphQLAuthCheck(context); - // Make sure that the page size is not too important - const first = args.first || DEFAULT_FIRST; - checkPageSize(first); - try { - const ability: AppAbility = context.user.ability; - const abilityFilters = PullJob.find( - accessibleBy(ability, 'read').PullJob - ).getFilter(); - const filters: any[] = [abilityFilters]; - - const afterCursor = args.afterCursor; - const cursorFilters = afterCursor - ? { - _id: { - $gt: decodeCursor(afterCursor), - }, - } - : {}; - - let items: any[] = await PullJob.find({ - $and: [cursorFilters, ...filters], - }) - .sort({ _id: 1 }) - .limit(first + 1); - - const hasNextPage = items.length > first; - if (hasNextPage) { - items = items.slice(0, items.length - 1); - } - const edges = items.map((r) => ({ - cursor: encodeCursor(r.id.toString()), - node: r, - })); - return { - pageInfo: { - hasNextPage, - startCursor: edges.length > 0 ? edges[0].cursor : null, - endCursor: edges.length > 0 ? edges[edges.length - 1].cursor : null, - }, - edges, - totalCount: await PullJob.countDocuments({ $and: filters }), - }; - } catch (err) { - logger.error(err.message, { stack: err.stack }); - if (err instanceof GraphQLError) { - throw new GraphQLError(err.message); - } - throw new GraphQLError( - context.i18next.t('common.errors.internalServerError') - ); - } - }, -}; diff --git a/src/schema/types/application.type.ts b/src/schema/types/application.type.ts index b3ea87232..8b1e77915 100644 --- a/src/schema/types/application.type.ts +++ b/src/schema/types/application.type.ts @@ -14,7 +14,6 @@ import { Channel, Application, PositionAttributeCategory, - PullJob, } from '@models'; import mongoose from 'mongoose'; import { @@ -23,7 +22,6 @@ import { RoleType, AccessType, PositionAttributeCategoryType, - PullJobType, TemplateType, DistributionListType, encodeCursor, @@ -448,17 +446,6 @@ export const ApplicationType = new GraphQLObjectType({ }, }, subscriptions: { type: new GraphQLList(SubscriptionType) }, - pullJobs: { - type: new GraphQLList(PullJobType), - async resolve(parent, args, context) { - const ability: AppAbility = context.user.ability; - const pullJobs = PullJob.find({ - ...accessibleBy(ability, 'read').PullJob, - _id: { $in: parent.pullJobs }, - }); - return pullJobs; - }, - }, permissions: { type: AccessType, resolve(parent, args, context) { diff --git a/src/schema/types/index.ts b/src/schema/types/index.ts index e87cccf86..5444273e0 100644 --- a/src/schema/types/index.ts +++ b/src/schema/types/index.ts @@ -9,7 +9,6 @@ export * from './page.type'; export * from './permission.type'; export * from './positionAttribute.type'; export * from './positionAttributeCategory.type'; -export * from './pullJob.type'; export * from './record.type'; export * from './referenceData.type'; export * from './resource.type'; diff --git a/src/schema/types/pullJob.type.ts b/src/schema/types/pullJob.type.ts deleted file mode 100644 index 921914a51..000000000 --- a/src/schema/types/pullJob.type.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { - GraphQLObjectType, - GraphQLID, - GraphQLString, - GraphQLList, -} from 'graphql'; -import GraphQLJSON from 'graphql-type-json'; -import { ApiConfiguration, Form, Channel } from '@models'; -import { StatusEnumType } from '@const/enumTypes'; -import { AppAbility } from '@security/defineUserAbility'; -import { ApiConfigurationType } from './apiConfiguration.type'; -import { ChannelType } from './channel.type'; -import { FormType } from './form.type'; -import { Connection } from './pagination.type'; -import { accessibleBy } from '@casl/mongoose'; - -/** GraphQL pull job type definition */ -export const PullJobType = new GraphQLObjectType({ - name: 'PullJob', - fields: () => ({ - id: { type: GraphQLID }, - name: { type: GraphQLString }, - status: { type: StatusEnumType }, - apiConfiguration: { - type: ApiConfigurationType, - async resolve(parent, args, context) { - const ability: AppAbility = context.user.ability; - const apiConfig = await ApiConfiguration.findOne({ - _id: parent.apiConfiguration, - ...accessibleBy(ability, 'read').ApiConfiguration, - }); - return apiConfig; - }, - }, - url: { type: GraphQLString }, - path: { type: GraphQLString }, - schedule: { type: GraphQLString }, - convertTo: { - type: FormType, - async resolve(parent, args, context) { - const ability: AppAbility = context.user.ability; - const form = await Form.findOne({ - _id: parent.convertTo, - ...accessibleBy(ability, 'read').Form, - }); - return form; - }, - }, - mapping: { type: GraphQLJSON }, - uniqueIdentifiers: { type: new GraphQLList(GraphQLString) }, - channel: { - type: ChannelType, - async resolve(parent, args, context) { - const ability: AppAbility = context.user.ability; - const channel = await Channel.findOne({ - _id: parent.channel, - ...accessibleBy(ability, 'read').Channel, - }); - return channel; - }, - }, - }), -}); - -/** GraphQL pull job connection type definition */ -export const PullJobConnectionType = Connection(PullJobType); diff --git a/src/security/defineUserAbility.ts b/src/security/defineUserAbility.ts index 3e4072cfb..c389337c5 100644 --- a/src/security/defineUserAbility.ts +++ b/src/security/defineUserAbility.ts @@ -27,7 +27,6 @@ import { User, Version, Workflow, - PullJob, ReferenceData, Group, Template, @@ -59,7 +58,6 @@ type Models = | Notification | Page | Permission - | PullJob | Record | ReferenceData | Resource @@ -318,12 +316,12 @@ export default function defineUserAbility(user: User | Client): AppAbility { }); /* === - Creation / Access / Edition / Deletion of API configurations, PullJobs and ReferenceData + Creation / Access / Edition / Deletion of API configurations and ReferenceData === */ if (userGlobalPermissions.includes(permissions.canManageApiConfigurations)) { can( ['create', 'read', 'update', 'delete'], - ['ApiConfiguration', 'PullJob', 'ReferenceData'] + ['ApiConfiguration', 'ReferenceData'] ); } else { // can('read', 'ApiConfiguration', filters('canSee', user)); diff --git a/src/server/EIOSOwnernshipMapping.ts b/src/server/EIOSOwnernshipMapping.ts deleted file mode 100644 index 286817832..000000000 --- a/src/server/EIOSOwnernshipMapping.ts +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable prettier/prettier */ -/** - * JSON used to map ownership for EIOS Pulljob - */ -export const ownershipMappingJSON = { - '0_COVID Countries': ['Signal HQ PHI','Signal HQ AEE'], - 'AEE_Monkeypox_Expanded': ['Signal HQ PHI','Signal HQ AEE'], - 'AFRO Media Monitoring w/o COVID-19': ['Signal HQ PHI','Signal AFRO'], - 'AFRO Signals': ['Signal HQ PHI','Signal AFRO'], - 'AFRO: EVD/VHF East African monitoring board 2022': ['Signal HQ PHI','Signal AFRO'], - 'AFRO_IDSR priority diseases and conditions': ['Signal HQ PHI','Signal AFRO'], - 'Bahrain, Jordan and Lebanon Cov': ['Signal HQ PHI','Signal HQ AEE'], - 'Balkans_COVID_public': ['Signal HQ PHI','Signal EURO'], - 'Caucasus_COVID_public': ['Signal HQ PHI','Signal EURO'], - 'CentralAsia_COVID_public': ['Signal HQ PHI','Signal EURO'], - 'CentralEurope_COVID_public': ['Signal HQ PHI','Signal EURO'], - 'COVID_AdHoc_EURO_public': ['Signal HQ PHI','Signal EURO'], - 'COVID-19 Intel monitoring - GLOBAL': ['Signal HQ PHI','Signal HQ AEE'], - 'Ebola/MVD alert board': ['Signal HQ PHI',], - 'ELR ADHOC': ['Signal HQ PHI','Signal HQ AEE'], - 'ELR-Samuel': ['Signal HQ PHI','Signal HQ AEE'], - 'EMRO COVID Vaccination': ['Signal HQ PHI','Signal EMRO'], - 'EMRO- Excl nCoV': ['Signal HQ PHI','Signal EMRO'], - 'EMRO new Variant': ['Signal HQ PHI','Signal EMRO'], - 'EUROHIM_WEST_public': ['Signal HQ PHI','Signal EURO'], - 'MM_EURO_SignalApp_Pilot': ['Signal HQ PHI','Signal EURO'], - 'Monkeypox - Global Outbreak': ['Signal HQ PHI','Signal HQ AEE'], - 'Monkeypox Global Outbreak': ['Signal HQ PHI','Signal HQ AEE'], - 'Monkeypox_Global EBS_Expanded': ['Signal HQ PHI','Signal HQ AEE'], - 'NRBC - Sahel': ['Signal HQ PHI','Signal HQ FCV'], - 'PAHO - Natural Disasters': ['Signal HQ PHI','Signal AMRO'], - 'PAHO - Priority Hazards': ['Signal HQ PHI','Signal AMRO'], - 'Pilot - Daily Monitoring': ['Signal HQ PHI',], - 'Priority disease Global - excluding COVID-19 and Mpox category definition': - ['Signal HQ PHI'], - 'Priority disease Global - excluding COVID-19 category definition': ['Signal HQ PHI',], - 'Priority disease Global - excluding nCoV and monkeypox category definition': - ['Signal HQ PHI'], - 'Priority disease Global - not including nCoV and monkeypox category definition': - ['Signal HQ PHI'], - 'Priority diseases Global - not including coronavirus & Monkeypox category definition': - ['Signal HQ PHI'], - 'Priority diseases Global - not including coronavirus category definition': - ['Signal HQ PHI'], - 'Priority Hazards': ['Signal HQ PHI',], - 'Priority sources - excluding COVID-19': ['Signal HQ PHI',], - 'Priority sources - excluding COVID-19 and mpox': ['Signal HQ PHI',], - 'Priority sources - excluding nCoV and monkeypox': ['Signal HQ PHI',], - 'Priority sources - not including coronavirus & Monkeypox category definition': - ['Signal HQ PHI'], - 'Priority sources - not including coronavirus category definition': ['Signal HQ PHI',], - 'qa': ['Signal HQ PHI','Signal EMRO'], - 'qa - euro': ['Signal HQ PHI','Signal EURO'], - 'QA- HQ- AFRO - EURO- AMRO': ['Signal HQ PHI','Signal EURO', 'Signal AFRO', 'Signal AMRO'], - 'qa4 - AFRO- AMRO- EURO': ['Signal HQ PHI','Signal EURO', 'Signal AFRO', 'Signal AMRO'], - 'SARS-CoV-2 variants': ['Signal HQ PHI','Signal HQ AEE'], - 'SEAR Signals For Export': ['Signal HQ PHI','Signal SEARO'], - 'Shared Ukraine board': ['Signal HQ PHI','Signal EURO'], - 'Signals for Follow Up- SEAR': ['Signal HQ PHI','Signal SEARO'], - 'Sudan virus disease Uganda - October 2022': ['Signal HQ PHI',], - 'TestHWF20210322': ['Signal HQ PHI',], - 'Ukraine': ['Signal HQ PHI','Signal EURO'], - 'WPRO - Disaster': ['Signal HQ PHI','Signal WPRO'], - 'WPRO Disease 2 - Not nCoV': ['Signal HQ PHI','Signal WPRO'], -}; diff --git a/src/server/pullJobScheduler.ts b/src/server/pullJobScheduler.ts deleted file mode 100644 index 30710da32..000000000 --- a/src/server/pullJobScheduler.ts +++ /dev/null @@ -1,670 +0,0 @@ -import { authType } from '@const/enumTypes'; -import { - BASE_PLACEHOLDER_REGEX, - extractStringFromBrackets, -} from '../const/placeholders'; -import { - ApiConfiguration, - Form, - Notification, - PullJob, - Record as RecordModel, - User, - Role, -} from '@models'; -import pubsub from './pubsub'; -import { CronJob } from 'cron'; -// import * as CryptoJS from 'crypto-js'; -import mongoose from 'mongoose'; -import { getToken } from '@utils/proxy'; -import { getNextId, transformRecord } from '@utils/form'; -import { logger } from '../services/logger.service'; -import * as cronValidator from 'cron-validator'; -import get from 'lodash/get'; -import axios from 'axios'; -import { ownershipMappingJSON } from './EIOSOwnernshipMapping'; - -/** A map with the task ids as keys and the scheduled tasks as values */ -const taskMap: Record = {}; - -/** Record's default fields */ -const DEFAULT_FIELDS = ['createdBy']; - -/** - * Dynamically building the list of Signal Apps names for EIOS - */ -const EIOS_APP_NAMES: string[] = [ - ...new Set( // Remove duplicate values - Object.values(ownershipMappingJSON).reduce((prev, curr) => { - prev.push(...curr); // Push all the Apps names into an array - return prev; - }, []) - ), -]; - -/** - * Global function called on server start to initialize all the pullJobs. - */ -const pullJobScheduler = async () => { - const pullJobs = await PullJob.find({ status: 'active' }).populate({ - path: 'apiConfiguration', - model: 'ApiConfiguration', - }); - - for (const pullJob of pullJobs) { - // eslint-disable-next-line @typescript-eslint/no-use-before-define - scheduleJob(pullJob); - } -}; - -export default pullJobScheduler; - -/** - * Schedule or re-schedule a pullJob. - * - * @param pullJob pull job to schedule - */ -export const scheduleJob = (pullJob: PullJob) => { - try { - const task = taskMap[pullJob.id]; - if (task) { - task.stop(); - } - const schedule = get(pullJob, 'schedule', ''); - if (cronValidator.isValidCron(schedule)) { - taskMap[pullJob.id] = new CronJob( - pullJob.schedule, - async () => { - logger.info('📥 Starting a pull from job ' + pullJob.name); - const apiConfiguration: ApiConfiguration = pullJob.apiConfiguration; - try { - if (apiConfiguration.authType === authType.serviceToService) { - // Decrypt settings - // const settings: { - // authTargetUrl: string; - // apiClientID: string; - // safeSecret: string; - // scope: string; - // } = JSON.parse( - // CryptoJS.AES.decrypt( - // apiConfiguration.settings, - // config.get('encryption.key') - // ).toString(CryptoJS.enc.Utf8) - // ); - - // Get auth token and start pull Logic - const token: string = await getToken(apiConfiguration); - // eslint-disable-next-line @typescript-eslint/no-use-before-define - fetchRecordsServiceToService(pullJob, token); - } - if (apiConfiguration.authType === authType.public) { - // eslint-disable-next-line @typescript-eslint/no-use-before-define - fetchRecordsPublic(pullJob); - } - if (apiConfiguration.authType === authType.authorizationCode) { - throw new Error( - 'Unsupported Api configuration with Authorization Code authentication.' - ); - } - } catch (err) { - logger.error(err.message, { stack: err.stack }); - } - }, - null, - true - ); - logger.info('📅 Scheduled job ' + pullJob.name); - } else { - throw new Error(`[${pullJob.name}] Invalid schedule: ${schedule}`); - } - } catch (err) { - logger.error(err.message); - } -}; - -/** - * Unschedule an existing pullJob from its id. - * - * @param pullJob pull job to unschedule - */ -export const unscheduleJob = (pullJob: PullJob): void => { - const task = taskMap[pullJob.id]; - if (task) { - task.stop(); - logger.info( - `📆 Unscheduled job ${pullJob.name ? pullJob.name : pullJob.id}` - ); - } -}; - -/** - * Fetch records using the hardcoded workflow for service-to-service API type (EIOS). - * - * @param pullJob pull job configuration to use - * @param token authentication token - */ -const fetchRecordsServiceToService = ( - pullJob: PullJob, - token: string -): void => { - const apiConfiguration: ApiConfiguration = pullJob.apiConfiguration; - // Hard coded for EIOS due to specific behavior - const EIOS_ORIGIN = 'https://portal.who.int/eios/'; - // === HARD CODED ENDPOINTS === - const headers: any = { - Authorization: 'Bearer ' + token, - }; - // Hardcoded specific behavior for EIOS - if (apiConfiguration.endpoint.startsWith(EIOS_ORIGIN)) { - // === HARD CODED ENDPOINTS === - const boardsUrl = 'GetBoards?tags=signal+app'; - const articlesUrl = 'GetPinnedArticles'; - axios({ - url: apiConfiguration.endpoint + boardsUrl, - method: 'get', - headers, - }) - .then(({ data }) => { - if (data && data.result) { - const boardIds = data.result.map((x) => x.id); - axios({ - url: `${apiConfiguration.endpoint}${articlesUrl}?boardIds=${boardIds}`, - method: 'get', - headers, - }) - .then(({ data: data2 }) => { - if (data2 && data2.result) { - // eslint-disable-next-line @typescript-eslint/no-use-before-define - insertRecords(data2.result, pullJob, true, false); - } - }) - .catch((err) => { - logger.error( - `Job ${pullJob.name} : Failed to get pinned articles : ${err}` - ); - }); - } - }) - .catch((err) => { - logger.error( - `Job ${pullJob.name} : Failed to get signal app boards : ${err}` - ); - }); - } else { - // Generic case - axios({ - url: apiConfiguration.endpoint + pullJob.url, - method: 'get', - headers, - }) - .then(({ data }) => { - const records = pullJob.path ? get(data, pullJob.path) : data; - if (records) { - // eslint-disable-next-line @typescript-eslint/no-use-before-define - insertRecords(records, pullJob, false, false); - } - }) - .catch((err) => { - logger.error(`Job ${pullJob.name} : Failed to fetch data : ${err}`); - }); - } -}; - -/** - * Fetch records using the generic workflow for public endpoints. - * - * @param pullJob pull job to use - */ -const fetchRecordsPublic = (pullJob: PullJob): void => { - const apiConfiguration: ApiConfiguration = pullJob.apiConfiguration; - logger.info(`Execute pull job operation: ${pullJob.name}`); - axios({ - url: apiConfiguration.endpoint + pullJob.url, - method: 'get', - }) - .then(({ data }) => { - const records = pullJob.path ? get(data, pullJob.path) : data; - if (records) { - // eslint-disable-next-line @typescript-eslint/no-use-before-define - insertRecords(records, pullJob, false, false); - } - }) - .catch((err) => { - logger.error(`Job ${pullJob.name} : Failed to fetch data : ${err}`); - }); -}; - -/** - * Access property of passed object including nested properties and map properties on array if needed. - * - * @param data object to get property in - * @param identifier path - * @returns property - */ -const accessFieldIncludingNested = (data: any, identifier: string): any => { - if (identifier.includes('.')) { - // Loop to access nested elements if we have . - const fields: any[] = identifier.split('.'); - const firstField = fields.shift(); - let value = data[firstField]; - for (const field of fields) { - if (value) { - if (Array.isArray(value) && isNaN(field)) { - value = value.flatMap((x) => (x ? x[field] : null)); - } else { - value = value[field]; - } - } else { - return null; - } - } - return value; - } else { - // Map to corresponding property - return data[identifier]; - } -}; - -/** - * Get Mongo Filters to get user role for a specific application - * - * @param appName Name of the application - * @returns List of Mongo filters - */ -const getUserRoleFiltersFromApp = (appName: string): any => { - return [ - { - $lookup: { - from: 'applications', - localField: 'application', - foreignField: '_id', - as: '_application', - }, - }, - { - $match: { - $and: [ - { title: 'User' }, - { _application: { $elemMatch: { name: appName } } }, - ], - }, - }, - ]; -}; - -/** - * Use the fetched data to insert records into the dB if needed. - * - * @param data array of data fetched from API - * @param pullJob pull job configuration - * @param fromRoute tells if the insertion is done from pull-job or route - * @param isEIOS is EIOS pulljob or not - */ -export const insertRecords = async ( - data: any[], - pullJob: PullJob, - isEIOS = false, - fromRoute?: boolean -): Promise => { - const form = await Form.findById(pullJob.convertTo); - if (form) { - const records = []; - const unicityConditions = pullJob.uniqueIdentifiers; - // Map unicity conditions to check if we already have some corresponding records in the DB - const mappedUnicityConditions = unicityConditions.map((x) => - Object.keys(pullJob.mapping).find((key) => pullJob.mapping[key] === x) - ); - // Initialize the array of linked fields in the case we have an array unique identifier with linked fields - const linkedFieldsArray = new Array>( - unicityConditions.length - ); - const filters = []; - for (let elementIndex = 0; elementIndex < data.length; elementIndex++) { - const element = data[elementIndex]; - const filter = {}; - for ( - let unicityIndex = 0; - unicityIndex < unicityConditions.length; - unicityIndex++ - ) { - const identifier = unicityConditions[unicityIndex]; - const mappedIdentifier = mappedUnicityConditions[unicityIndex]; - // Check if it's an automatically generated element which already have some part of the identifiers set up - const value = - element[`__${identifier}`] === undefined - ? accessFieldIncludingNested(element, identifier) - : element[`__${identifier}`]; - // Prevent adding new records with identifier null, or type object or array with any at least one null value in it. - if ( - !value || - (typeof value === 'object' && - ((Array.isArray(value) && - value.some((x) => x === null || x === undefined)) || - !Array.isArray(value))) - ) { - element.__notValid = true; - // If a uniqueIdentifier value is an array, duplicate the element and add filter for the first one since the other will be handled in subsequent steps - } else if (Array.isArray(value)) { - // Get related fields from the mapping to duplicate use different values for these ones instead of concatenate everything - let linkedFields = linkedFieldsArray[unicityIndex]; - if (!linkedFields) { - // eslint-disable-next-line @typescript-eslint/no-use-before-define - linkedFields = getLinkedFields( - identifier, - pullJob.mapping, - element - ); - linkedFieldsArray[unicityIndex] = linkedFields; - } - const linkedValues = new Array(linkedFields.length); - for ( - let linkedIndex = 0; - linkedIndex < linkedFields.length; - linkedIndex++ - ) { - // eslint-disable-next-line @typescript-eslint/no-use-before-define - linkedValues[linkedIndex] = accessFieldIncludingNested( - element, - linkedFields[linkedIndex] - ); - } - for (let valueIndex = 0; valueIndex < value.length; valueIndex++) { - // Push new element if not the first one while setting identifier field and linked fields - if (valueIndex === 0) { - element[`__${identifier}`] = value[valueIndex]; - Object.assign(filter, { - $or: [ - { [`data.${mappedIdentifier}`]: value[valueIndex] }, - { - [`data.${mappedIdentifier}`]: value[valueIndex].toString(), - }, - ], - }); - for ( - let linkedIndex = 0; - linkedIndex < linkedFields.length; - linkedIndex++ - ) { - element[`__${linkedFields[linkedIndex]}`] = - linkedValues[linkedIndex][0]; - } - } else { - const newElement = Object.assign({}, element); - newElement[`__${identifier}`] = value[valueIndex]; - for ( - let linkedIndex = 0; - linkedIndex < linkedFields.length; - linkedIndex++ - ) { - newElement[`__${linkedFields[linkedIndex]}`] = - linkedValues[linkedIndex][valueIndex]; - } - data.splice(elementIndex + 1, 0, newElement); - } - } - } else { - element[`__${identifier}`] = value; - Object.assign(filter, { - $or: [ - { [`data.${mappedIdentifier}`]: value }, - { [`data.${mappedIdentifier}`]: value.toString() }, - ], - }); - } - } - filters.push(filter); - } - // Find records already existing if any - const selectedFields = mappedUnicityConditions.map((x) => `data.${x}`); - const duplicateRecords = await RecordModel.find({ - form: pullJob.convertTo, - $or: filters, - }).select(selectedFields); - - // If EIOS pullJob, build a mapping JSON to assign ownership (role ids) - const ownershipMappingWithIds: any = {}; - if (isEIOS) { - // Create a dictionary of user roles ids - const appRolesWithIds = {}; - const promisesStack = []; - EIOS_APP_NAMES.forEach((appName) => { - promisesStack.push( - Role.aggregate(getUserRoleFiltersFromApp(appName)).then( - (appUserRole) => { - if (appUserRole[0]) { - appRolesWithIds[appName] = appUserRole[0]._id; - } - } - ) - ); - }); - await Promise.allSettled(promisesStack); - - for (const [key, value] of Object.entries(ownershipMappingJSON)) { - ownershipMappingWithIds[key] = []; - if (value.length > 0) { - value.forEach((elt) => { - if (appRolesWithIds[elt]) { - ownershipMappingWithIds[key].push(appRolesWithIds[elt]); - } - }); - } - } - } - - for (const element of data) { - // eslint-disable-next-line @typescript-eslint/no-use-before-define - const mappedElement = mapData( - pullJob.mapping, - element, - unicityConditions.concat(linkedFieldsArray.flat()) - ); - // Adapt identifiers after mapping so if arrays are involved, it will correspond to each element of the array - for ( - let unicityIndex = 0; - unicityIndex < unicityConditions.length; - unicityIndex++ - ) { - const identifier = unicityConditions[unicityIndex]; - const mappedIdentifier = mappedUnicityConditions[unicityIndex]; - mappedElement[mappedIdentifier] = element[`__${identifier}`]; - // Adapt also linkedFields if any - const linkedFields = linkedFieldsArray[unicityIndex]; - if (linkedFields) { - // Storing already assigned fields in the case we have different fields mapped to the same path - const alreadyAssignedFields = []; - for (const linkedField of linkedFields) { - const mappedField = Object.keys(pullJob.mapping).find( - (key) => - pullJob.mapping[key] === linkedField && - !alreadyAssignedFields.includes(key) - ); - alreadyAssignedFields.push(mappedField); - mappedElement[mappedField] = element[`__${linkedField}`]; - } - } - } - // Check if element is already stored in the DB and if it has unique identifiers correctly set up - const isDuplicate = element.__notValid - ? true - : duplicateRecords.some((record) => { - for ( - let unicityIndex = 0; - unicityIndex < unicityConditions.length; - unicityIndex++ - ) { - const identifier = unicityConditions[unicityIndex]; - const mappedIdentifier = mappedUnicityConditions[unicityIndex]; - const recordValue = record.data[mappedIdentifier] || ''; - const elementValue = element[`__${identifier}`] || ''; - if (recordValue.toString() !== elementValue.toString()) { - return false; - } - } - return true; - }); - - if (isEIOS) { - // Assign correct ownership value based on mapping JSON and board name - const boardName = mappedElement.article_board_name; - mappedElement.ownership = - ownershipMappingWithIds[boardName].map(String); - } - // If everything is fine, push it in the array for saving - if (!isDuplicate) { - transformRecord(mappedElement, form.fields); - let record = new RecordModel({ - incrementalId: await getNextId( - String(form.resource ? form.resource : pullJob.convertTo) - ), - form: pullJob.convertTo, - data: mappedElement, - resource: form.resource ? form.resource : null, - _form: { - _id: form._id, - name: form.name, - }, - }) as RecordModel; - // eslint-disable-next-line @typescript-eslint/no-use-before-define - record = await setSpecialFields(record); - records.push(record); - } - } - let insertReportMessage = ''; - try { - const insertedRecords = await RecordModel.insertMany(records); - if (fromRoute) { - insertReportMessage = `${insertedRecords.length} new records of form "${form.name}" created from records insertion route`; - } else { - insertReportMessage = `${insertedRecords.length} new records of form "${form.name}" created from pulljob "${pullJob.name}"`; - } - logger.info(insertReportMessage); - if (pullJob.channel && records.length > 0) { - const notification = new Notification({ - action: insertReportMessage, - content: '', - createdAt: new Date(), - channel: pullJob.channel.toString(), - seenBy: [], - }); - await notification.save(); - const publisher = await pubsub(); - publisher.publish(pullJob.channel.toString(), { notification }); - } - return insertReportMessage; - } catch (err) { - return 'Record insertion failed'; - } - } else { - return 'Cannot find form with id ' + pullJob.convertTo; - } -}; - -/** - * Map the data retrieved so it match with the target Form. - * - * @param mapping mapping - * @param data data to map - * @param skippedIdentifiers keys to skip - * @returns mapped data - */ -export const mapData = ( - mapping: any, - data: any, - skippedIdentifiers?: string[] -): any => { - const out = {}; - if (mapping) { - for (const key of Object.keys(mapping)) { - const identifier: string = mapping[key]; - if (identifier.match(BASE_PLACEHOLDER_REGEX)) { - // Put the raw string passed if it's surrounded by double brackets - out[key] = extractStringFromBrackets(identifier); - } else { - // Skip identifiers overwrited in the next step (LinkedFields and UnicityConditions) - if (!skippedIdentifiers.includes(identifier)) { - // Access field - // eslint-disable-next-line @typescript-eslint/no-use-before-define - out[key] = accessFieldIncludingNested(data, identifier); - } - } - } - return out; - } else { - return data; - } -}; - -/** - * Get fields linked with the passed array identifiers because using a mapping on the same array. - * - * @param identifier key - * @param mapping mapping - * @param data data to insert - * @returns list of linked fields. - */ -const getLinkedFields = ( - identifier: string, - mapping: any, - data: any -): string[] => { - if (identifier.includes('.')) { - const fields: any[] = identifier.split('.'); - let identifierArrayKey = fields.shift(); - let value = data[identifierArrayKey]; - if (!Array.isArray(value)) { - for (const field of fields) { - identifierArrayKey += '.' + field; - if (value) { - if (Array.isArray(value) && isNaN(field)) { - value = false; - } else { - value = value[field]; - } - } - } - } - const linkedFields = []; - for (const key of Object.keys(mapping)) { - const externalKey = mapping[key]; - if ( - externalKey !== identifier && - externalKey.includes(identifierArrayKey) - ) { - linkedFields.push(externalKey); - } - } - return linkedFields; - } else { - return []; - } -}; - -/** - * If some specialFields are used in the mapping, set them at the right place in the record model. - * - * @param record new record - * @returns updated record. - */ -const setSpecialFields = async (record: RecordModel): Promise => { - const keys = Object.keys(record.data); - for (const key of keys) { - if (DEFAULT_FIELDS.includes(key)) { - switch (key) { - case 'createdBy': { - const username = record.data[key]; - const user = await User.findOne({ username }, 'id'); - if (user && user?.id) { - record.createdBy.user = new mongoose.Types.ObjectId(user._id); - delete record.data[key]; - } - break; - } - default: { - break; - } - } - } - } - return record; -}; From 370750bc2ffdd462767ce684b0ace0e36cddc6d6 Mon Sep 17 00:00:00 2001 From: RenzoPrats Date: Tue, 16 Jan 2024 12:23:41 -0300 Subject: [PATCH 2/5] lint fixed --- src/routes/upload/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/routes/upload/index.ts b/src/routes/upload/index.ts index 5dad56780..265b2c44e 100644 --- a/src/routes/upload/index.ts +++ b/src/routes/upload/index.ts @@ -17,7 +17,6 @@ import i18next from 'i18next'; import get from 'lodash/get'; import { logger } from '@services/logger.service'; import { accessibleBy } from '@casl/mongoose'; -import jwtDecode from 'jwt-decode'; /** File size limit, in bytes */ const FILE_SIZE_LIMIT = 7 * 1024 * 1024; From c7b6f1c9aab748eaecb1fa6339ea4362af4dc8f8 Mon Sep 17 00:00:00 2001 From: RenzoPrats Date: Tue, 16 Jan 2024 14:15:25 -0300 Subject: [PATCH 3/5] fix: restoring insertRecords function --- src/models/index.ts | 1 + src/models/pullJob.model.ts | 63 ++++ src/routes/upload/index.ts | 27 ++ src/server/EIOSOwnernshipMapping.ts | 65 ++++ src/server/pullJobScheduler.ts | 472 ++++++++++++++++++++++++++++ 5 files changed, 628 insertions(+) create mode 100644 src/models/pullJob.model.ts create mode 100644 src/server/EIOSOwnernshipMapping.ts create mode 100644 src/server/pullJobScheduler.ts diff --git a/src/models/index.ts b/src/models/index.ts index b90203b86..191acf18a 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -10,6 +10,7 @@ export * from './page.model'; export * from './permission.model'; export * from './positionAttribute.model'; export * from './positionAttributeCategory.model'; +export * from './pullJob.model'; export * from './record.model'; export * from './referenceData.model'; export * from './resource.model'; diff --git a/src/models/pullJob.model.ts b/src/models/pullJob.model.ts new file mode 100644 index 000000000..daac92649 --- /dev/null +++ b/src/models/pullJob.model.ts @@ -0,0 +1,63 @@ +import { AccessibleRecordModel, accessibleRecordsPlugin } from '@casl/mongoose'; +import mongoose, { Schema, Document } from 'mongoose'; +import { status } from '@const/enumTypes'; +import { ApiConfiguration } from './apiConfiguration.model'; +import * as cron from 'cron-validator'; + +/** Mongoose pull job schema declaration */ +const pullJobSchema = new Schema({ + name: String, + status: { + type: String, + enum: Object.values(status), + }, + apiConfiguration: { + type: mongoose.Schema.Types.ObjectId, + ref: 'ApiConfiguration', + }, + url: String, + path: String, + schedule: { + type: String, + validate: { + validator: function (value) { + return value ? cron.isValidCron(value) : false; + }, + }, + }, + convertTo: { + type: mongoose.Schema.Types.ObjectId, + ref: 'Form', + }, + mapping: mongoose.Schema.Types.Mixed, + uniqueIdentifiers: [String], + channel: { + type: mongoose.Schema.Types.ObjectId, + ref: 'Channel', + }, +}); + +pullJobSchema.index({ name: 1 }, { unique: true }); + +/** Pull job documents interface declaration */ +export interface PullJob extends Document { + kind: 'PullJob'; + name: string; + status: string; + apiConfiguration: ApiConfiguration; + url: string; + path: string; + schedule: string; + convertTo: string; + mapping: any; + uniqueIdentifiers: string[]; + channel: string; +} +pullJobSchema.plugin(accessibleRecordsPlugin); + +/** Mongoose pull job model definition */ +// eslint-disable-next-line @typescript-eslint/no-redeclare +export const PullJob = mongoose.model>( + 'PullJob', + pullJobSchema +); diff --git a/src/routes/upload/index.ts b/src/routes/upload/index.ts index 265b2c44e..63beb674c 100644 --- a/src/routes/upload/index.ts +++ b/src/routes/upload/index.ts @@ -17,6 +17,8 @@ import i18next from 'i18next'; import get from 'lodash/get'; import { logger } from '@services/logger.service'; import { accessibleBy } from '@casl/mongoose'; +import { insertRecords as insertRecordsPulljob } from '@server/pullJobScheduler'; +import jwtDecode from 'jwt-decode'; /** File size limit, in bytes */ const FILE_SIZE_LIMIT = 7 * 1024 * 1024; @@ -211,6 +213,31 @@ router.post('/resource/records/:id', async (req: any, res) => { } }); +/** + * Upload a list of records for a resource in json format + */ +router.post('/resource/insert', async (req: any, res) => { + try { + const authToken = req.headers.authorization.split(' ')[1]; + const decodedToken = jwtDecode(authToken) as any; + + // Block if connected with user to Service + if (!decodedToken.email && !decodedToken.name) { + const insertRecordsMessage = await insertRecordsPulljob( + req.body.records, + req.body.parameters, + true, + true + ); + return res.status(200).send(insertRecordsMessage); + } + return res.status(400).send(req.t('common.errors.permissionNotGranted')); + } catch (err) { + logger.error(err.message, { stack: err.stack }); + return res.status(500).send(req.t('common.errors.internalServerError')); + } +}); + /** * Import a list of users for an application from an uploaded xlsx file */ diff --git a/src/server/EIOSOwnernshipMapping.ts b/src/server/EIOSOwnernshipMapping.ts new file mode 100644 index 000000000..286817832 --- /dev/null +++ b/src/server/EIOSOwnernshipMapping.ts @@ -0,0 +1,65 @@ +/* eslint-disable prettier/prettier */ +/** + * JSON used to map ownership for EIOS Pulljob + */ +export const ownershipMappingJSON = { + '0_COVID Countries': ['Signal HQ PHI','Signal HQ AEE'], + 'AEE_Monkeypox_Expanded': ['Signal HQ PHI','Signal HQ AEE'], + 'AFRO Media Monitoring w/o COVID-19': ['Signal HQ PHI','Signal AFRO'], + 'AFRO Signals': ['Signal HQ PHI','Signal AFRO'], + 'AFRO: EVD/VHF East African monitoring board 2022': ['Signal HQ PHI','Signal AFRO'], + 'AFRO_IDSR priority diseases and conditions': ['Signal HQ PHI','Signal AFRO'], + 'Bahrain, Jordan and Lebanon Cov': ['Signal HQ PHI','Signal HQ AEE'], + 'Balkans_COVID_public': ['Signal HQ PHI','Signal EURO'], + 'Caucasus_COVID_public': ['Signal HQ PHI','Signal EURO'], + 'CentralAsia_COVID_public': ['Signal HQ PHI','Signal EURO'], + 'CentralEurope_COVID_public': ['Signal HQ PHI','Signal EURO'], + 'COVID_AdHoc_EURO_public': ['Signal HQ PHI','Signal EURO'], + 'COVID-19 Intel monitoring - GLOBAL': ['Signal HQ PHI','Signal HQ AEE'], + 'Ebola/MVD alert board': ['Signal HQ PHI',], + 'ELR ADHOC': ['Signal HQ PHI','Signal HQ AEE'], + 'ELR-Samuel': ['Signal HQ PHI','Signal HQ AEE'], + 'EMRO COVID Vaccination': ['Signal HQ PHI','Signal EMRO'], + 'EMRO- Excl nCoV': ['Signal HQ PHI','Signal EMRO'], + 'EMRO new Variant': ['Signal HQ PHI','Signal EMRO'], + 'EUROHIM_WEST_public': ['Signal HQ PHI','Signal EURO'], + 'MM_EURO_SignalApp_Pilot': ['Signal HQ PHI','Signal EURO'], + 'Monkeypox - Global Outbreak': ['Signal HQ PHI','Signal HQ AEE'], + 'Monkeypox Global Outbreak': ['Signal HQ PHI','Signal HQ AEE'], + 'Monkeypox_Global EBS_Expanded': ['Signal HQ PHI','Signal HQ AEE'], + 'NRBC - Sahel': ['Signal HQ PHI','Signal HQ FCV'], + 'PAHO - Natural Disasters': ['Signal HQ PHI','Signal AMRO'], + 'PAHO - Priority Hazards': ['Signal HQ PHI','Signal AMRO'], + 'Pilot - Daily Monitoring': ['Signal HQ PHI',], + 'Priority disease Global - excluding COVID-19 and Mpox category definition': + ['Signal HQ PHI'], + 'Priority disease Global - excluding COVID-19 category definition': ['Signal HQ PHI',], + 'Priority disease Global - excluding nCoV and monkeypox category definition': + ['Signal HQ PHI'], + 'Priority disease Global - not including nCoV and monkeypox category definition': + ['Signal HQ PHI'], + 'Priority diseases Global - not including coronavirus & Monkeypox category definition': + ['Signal HQ PHI'], + 'Priority diseases Global - not including coronavirus category definition': + ['Signal HQ PHI'], + 'Priority Hazards': ['Signal HQ PHI',], + 'Priority sources - excluding COVID-19': ['Signal HQ PHI',], + 'Priority sources - excluding COVID-19 and mpox': ['Signal HQ PHI',], + 'Priority sources - excluding nCoV and monkeypox': ['Signal HQ PHI',], + 'Priority sources - not including coronavirus & Monkeypox category definition': + ['Signal HQ PHI'], + 'Priority sources - not including coronavirus category definition': ['Signal HQ PHI',], + 'qa': ['Signal HQ PHI','Signal EMRO'], + 'qa - euro': ['Signal HQ PHI','Signal EURO'], + 'QA- HQ- AFRO - EURO- AMRO': ['Signal HQ PHI','Signal EURO', 'Signal AFRO', 'Signal AMRO'], + 'qa4 - AFRO- AMRO- EURO': ['Signal HQ PHI','Signal EURO', 'Signal AFRO', 'Signal AMRO'], + 'SARS-CoV-2 variants': ['Signal HQ PHI','Signal HQ AEE'], + 'SEAR Signals For Export': ['Signal HQ PHI','Signal SEARO'], + 'Shared Ukraine board': ['Signal HQ PHI','Signal EURO'], + 'Signals for Follow Up- SEAR': ['Signal HQ PHI','Signal SEARO'], + 'Sudan virus disease Uganda - October 2022': ['Signal HQ PHI',], + 'TestHWF20210322': ['Signal HQ PHI',], + 'Ukraine': ['Signal HQ PHI','Signal EURO'], + 'WPRO - Disaster': ['Signal HQ PHI','Signal WPRO'], + 'WPRO Disease 2 - Not nCoV': ['Signal HQ PHI','Signal WPRO'], +}; diff --git a/src/server/pullJobScheduler.ts b/src/server/pullJobScheduler.ts new file mode 100644 index 000000000..af6f9ea4d --- /dev/null +++ b/src/server/pullJobScheduler.ts @@ -0,0 +1,472 @@ +import { + BASE_PLACEHOLDER_REGEX, + extractStringFromBrackets, +} from '../const/placeholders'; +import { + Form, + Notification, + PullJob, + Record as RecordModel, + User, + Role, +} from '@models'; +import pubsub from './pubsub'; +import { CronJob } from 'cron'; +// import * as CryptoJS from 'crypto-js'; +import mongoose from 'mongoose'; +import { getNextId, transformRecord } from '@utils/form'; +import { logger } from '../services/logger.service'; +import { ownershipMappingJSON } from './EIOSOwnernshipMapping'; + +/** A map with the task ids as keys and the scheduled tasks as values */ +const taskMap: Record = {}; + +/** Record's default fields */ +const DEFAULT_FIELDS = ['createdBy']; + +/** + * Dynamically building the list of Signal Apps names for EIOS + */ +const EIOS_APP_NAMES: string[] = [ + ...new Set( // Remove duplicate values + Object.values(ownershipMappingJSON).reduce((prev, curr) => { + prev.push(...curr); // Push all the Apps names into an array + return prev; + }, []) + ), +]; + +/** + * Access property of passed object including nested properties and map properties on array if needed. + * + * @param data object to get property in + * @param identifier path + * @returns property + */ +const accessFieldIncludingNested = (data: any, identifier: string): any => { + if (identifier.includes('.')) { + // Loop to access nested elements if we have . + const fields: any[] = identifier.split('.'); + const firstField = fields.shift(); + let value = data[firstField]; + for (const field of fields) { + if (value) { + if (Array.isArray(value) && isNaN(field)) { + value = value.flatMap((x) => (x ? x[field] : null)); + } else { + value = value[field]; + } + } else { + return null; + } + } + return value; + } else { + // Map to corresponding property + return data[identifier]; + } +}; + +/** + * Get Mongo Filters to get user role for a specific application + * + * @param appName Name of the application + * @returns List of Mongo filters + */ +const getUserRoleFiltersFromApp = (appName: string): any => { + return [ + { + $lookup: { + from: 'applications', + localField: 'application', + foreignField: '_id', + as: '_application', + }, + }, + { + $match: { + $and: [ + { title: 'User' }, + { _application: { $elemMatch: { name: appName } } }, + ], + }, + }, + ]; +}; + +/** + * Use the fetched data to insert records into the dB if needed. + * + * @param data array of data fetched from API + * @param pullJob pull job configuration + * @param fromRoute tells if the insertion is done from pull-job or route + * @param isEIOS is EIOS pulljob or not + */ +export const insertRecords = async ( + data: any[], + pullJob: PullJob, + isEIOS = false, + fromRoute?: boolean +): Promise => { + const form = await Form.findById(pullJob.convertTo); + if (form) { + const records = []; + const unicityConditions = pullJob.uniqueIdentifiers; + // Map unicity conditions to check if we already have some corresponding records in the DB + const mappedUnicityConditions = unicityConditions.map((x) => + Object.keys(pullJob.mapping).find((key) => pullJob.mapping[key] === x) + ); + // Initialize the array of linked fields in the case we have an array unique identifier with linked fields + const linkedFieldsArray = new Array>( + unicityConditions.length + ); + const filters = []; + for (let elementIndex = 0; elementIndex < data.length; elementIndex++) { + const element = data[elementIndex]; + const filter = {}; + for ( + let unicityIndex = 0; + unicityIndex < unicityConditions.length; + unicityIndex++ + ) { + const identifier = unicityConditions[unicityIndex]; + const mappedIdentifier = mappedUnicityConditions[unicityIndex]; + // Check if it's an automatically generated element which already have some part of the identifiers set up + const value = + element[`__${identifier}`] === undefined + ? accessFieldIncludingNested(element, identifier) + : element[`__${identifier}`]; + // Prevent adding new records with identifier null, or type object or array with any at least one null value in it. + if ( + !value || + (typeof value === 'object' && + ((Array.isArray(value) && + value.some((x) => x === null || x === undefined)) || + !Array.isArray(value))) + ) { + element.__notValid = true; + // If a uniqueIdentifier value is an array, duplicate the element and add filter for the first one since the other will be handled in subsequent steps + } else if (Array.isArray(value)) { + // Get related fields from the mapping to duplicate use different values for these ones instead of concatenate everything + let linkedFields = linkedFieldsArray[unicityIndex]; + if (!linkedFields) { + // eslint-disable-next-line @typescript-eslint/no-use-before-define + linkedFields = getLinkedFields( + identifier, + pullJob.mapping, + element + ); + linkedFieldsArray[unicityIndex] = linkedFields; + } + const linkedValues = new Array(linkedFields.length); + for ( + let linkedIndex = 0; + linkedIndex < linkedFields.length; + linkedIndex++ + ) { + // eslint-disable-next-line @typescript-eslint/no-use-before-define + linkedValues[linkedIndex] = accessFieldIncludingNested( + element, + linkedFields[linkedIndex] + ); + } + for (let valueIndex = 0; valueIndex < value.length; valueIndex++) { + // Push new element if not the first one while setting identifier field and linked fields + if (valueIndex === 0) { + element[`__${identifier}`] = value[valueIndex]; + Object.assign(filter, { + $or: [ + { [`data.${mappedIdentifier}`]: value[valueIndex] }, + { + [`data.${mappedIdentifier}`]: value[valueIndex].toString(), + }, + ], + }); + for ( + let linkedIndex = 0; + linkedIndex < linkedFields.length; + linkedIndex++ + ) { + element[`__${linkedFields[linkedIndex]}`] = + linkedValues[linkedIndex][0]; + } + } else { + const newElement = Object.assign({}, element); + newElement[`__${identifier}`] = value[valueIndex]; + for ( + let linkedIndex = 0; + linkedIndex < linkedFields.length; + linkedIndex++ + ) { + newElement[`__${linkedFields[linkedIndex]}`] = + linkedValues[linkedIndex][valueIndex]; + } + data.splice(elementIndex + 1, 0, newElement); + } + } + } else { + element[`__${identifier}`] = value; + Object.assign(filter, { + $or: [ + { [`data.${mappedIdentifier}`]: value }, + { [`data.${mappedIdentifier}`]: value.toString() }, + ], + }); + } + } + filters.push(filter); + } + // Find records already existing if any + const selectedFields = mappedUnicityConditions.map((x) => `data.${x}`); + const duplicateRecords = await RecordModel.find({ + form: pullJob.convertTo, + $or: filters, + }).select(selectedFields); + + // If EIOS pullJob, build a mapping JSON to assign ownership (role ids) + const ownershipMappingWithIds: any = {}; + if (isEIOS) { + // Create a dictionary of user roles ids + const appRolesWithIds = {}; + const promisesStack = []; + EIOS_APP_NAMES.forEach((appName) => { + promisesStack.push( + Role.aggregate(getUserRoleFiltersFromApp(appName)).then( + (appUserRole) => { + if (appUserRole[0]) { + appRolesWithIds[appName] = appUserRole[0]._id; + } + } + ) + ); + }); + await Promise.allSettled(promisesStack); + + for (const [key, value] of Object.entries(ownershipMappingJSON)) { + ownershipMappingWithIds[key] = []; + if (value.length > 0) { + value.forEach((elt) => { + if (appRolesWithIds[elt]) { + ownershipMappingWithIds[key].push(appRolesWithIds[elt]); + } + }); + } + } + } + + for (const element of data) { + // eslint-disable-next-line @typescript-eslint/no-use-before-define + const mappedElement = mapData( + pullJob.mapping, + element, + unicityConditions.concat(linkedFieldsArray.flat()) + ); + // Adapt identifiers after mapping so if arrays are involved, it will correspond to each element of the array + for ( + let unicityIndex = 0; + unicityIndex < unicityConditions.length; + unicityIndex++ + ) { + const identifier = unicityConditions[unicityIndex]; + const mappedIdentifier = mappedUnicityConditions[unicityIndex]; + mappedElement[mappedIdentifier] = element[`__${identifier}`]; + // Adapt also linkedFields if any + const linkedFields = linkedFieldsArray[unicityIndex]; + if (linkedFields) { + // Storing already assigned fields in the case we have different fields mapped to the same path + const alreadyAssignedFields = []; + for (const linkedField of linkedFields) { + const mappedField = Object.keys(pullJob.mapping).find( + (key) => + pullJob.mapping[key] === linkedField && + !alreadyAssignedFields.includes(key) + ); + alreadyAssignedFields.push(mappedField); + mappedElement[mappedField] = element[`__${linkedField}`]; + } + } + } + // Check if element is already stored in the DB and if it has unique identifiers correctly set up + const isDuplicate = element.__notValid + ? true + : duplicateRecords.some((record) => { + for ( + let unicityIndex = 0; + unicityIndex < unicityConditions.length; + unicityIndex++ + ) { + const identifier = unicityConditions[unicityIndex]; + const mappedIdentifier = mappedUnicityConditions[unicityIndex]; + const recordValue = record.data[mappedIdentifier] || ''; + const elementValue = element[`__${identifier}`] || ''; + if (recordValue.toString() !== elementValue.toString()) { + return false; + } + } + return true; + }); + + if (isEIOS) { + // Assign correct ownership value based on mapping JSON and board name + const boardName = mappedElement.article_board_name; + mappedElement.ownership = + ownershipMappingWithIds[boardName].map(String); + } + // If everything is fine, push it in the array for saving + if (!isDuplicate) { + transformRecord(mappedElement, form.fields); + let record = new RecordModel({ + incrementalId: await getNextId( + String(form.resource ? form.resource : pullJob.convertTo) + ), + form: pullJob.convertTo, + data: mappedElement, + resource: form.resource ? form.resource : null, + _form: { + _id: form._id, + name: form.name, + }, + }) as RecordModel; + // eslint-disable-next-line @typescript-eslint/no-use-before-define + record = await setSpecialFields(record); + records.push(record); + } + } + let insertReportMessage = ''; + try { + const insertedRecords = await RecordModel.insertMany(records); + if (fromRoute) { + insertReportMessage = `${insertedRecords.length} new records of form "${form.name}" created from records insertion route`; + } else { + insertReportMessage = `${insertedRecords.length} new records of form "${form.name}" created from pulljob "${pullJob.name}"`; + } + logger.info(insertReportMessage); + if (pullJob.channel && records.length > 0) { + const notification = new Notification({ + action: insertReportMessage, + content: '', + createdAt: new Date(), + channel: pullJob.channel.toString(), + seenBy: [], + }); + await notification.save(); + const publisher = await pubsub(); + publisher.publish(pullJob.channel.toString(), { notification }); + } + return insertReportMessage; + } catch (err) { + return 'Record insertion failed'; + } + } else { + return 'Cannot find form with id ' + pullJob.convertTo; + } +}; + +/** + * Map the data retrieved so it match with the target Form. + * + * @param mapping mapping + * @param data data to map + * @param skippedIdentifiers keys to skip + * @returns mapped data + */ +export const mapData = ( + mapping: any, + data: any, + skippedIdentifiers?: string[] +): any => { + const out = {}; + if (mapping) { + for (const key of Object.keys(mapping)) { + const identifier: string = mapping[key]; + if (identifier.match(BASE_PLACEHOLDER_REGEX)) { + // Put the raw string passed if it's surrounded by double brackets + out[key] = extractStringFromBrackets(identifier); + } else { + // Skip identifiers overwrited in the next step (LinkedFields and UnicityConditions) + if (!skippedIdentifiers.includes(identifier)) { + // Access field + // eslint-disable-next-line @typescript-eslint/no-use-before-define + out[key] = accessFieldIncludingNested(data, identifier); + } + } + } + return out; + } else { + return data; + } +}; + +/** + * Get fields linked with the passed array identifiers because using a mapping on the same array. + * + * @param identifier key + * @param mapping mapping + * @param data data to insert + * @returns list of linked fields. + */ +const getLinkedFields = ( + identifier: string, + mapping: any, + data: any +): string[] => { + if (identifier.includes('.')) { + const fields: any[] = identifier.split('.'); + let identifierArrayKey = fields.shift(); + let value = data[identifierArrayKey]; + if (!Array.isArray(value)) { + for (const field of fields) { + identifierArrayKey += '.' + field; + if (value) { + if (Array.isArray(value) && isNaN(field)) { + value = false; + } else { + value = value[field]; + } + } + } + } + const linkedFields = []; + for (const key of Object.keys(mapping)) { + const externalKey = mapping[key]; + if ( + externalKey !== identifier && + externalKey.includes(identifierArrayKey) + ) { + linkedFields.push(externalKey); + } + } + return linkedFields; + } else { + return []; + } +}; + +/** + * If some specialFields are used in the mapping, set them at the right place in the record model. + * + * @param record new record + * @returns updated record. + */ +const setSpecialFields = async (record: RecordModel): Promise => { + const keys = Object.keys(record.data); + for (const key of keys) { + if (DEFAULT_FIELDS.includes(key)) { + switch (key) { + case 'createdBy': { + const username = record.data[key]; + const user = await User.findOne({ username }, 'id'); + if (user && user?.id) { + record.createdBy.user = new mongoose.Types.ObjectId(user._id); + delete record.data[key]; + } + break; + } + default: { + break; + } + } + } + } + return record; +}; From 2334a76282686f5ac97d00045186b8b4a9d62022 Mon Sep 17 00:00:00 2001 From: RenzoPrats Date: Tue, 16 Jan 2024 14:16:23 -0300 Subject: [PATCH 4/5] lint fixed --- src/server/pullJobScheduler.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/server/pullJobScheduler.ts b/src/server/pullJobScheduler.ts index af6f9ea4d..8e6595833 100644 --- a/src/server/pullJobScheduler.ts +++ b/src/server/pullJobScheduler.ts @@ -18,9 +18,6 @@ import { getNextId, transformRecord } from '@utils/form'; import { logger } from '../services/logger.service'; import { ownershipMappingJSON } from './EIOSOwnernshipMapping'; -/** A map with the task ids as keys and the scheduled tasks as values */ -const taskMap: Record = {}; - /** Record's default fields */ const DEFAULT_FIELDS = ['createdBy']; From 95dd1860eac3b9864917195d424a1ca5ec1dff1e Mon Sep 17 00:00:00 2001 From: RenzoPrats Date: Tue, 16 Jan 2024 14:18:56 -0300 Subject: [PATCH 5/5] lint fixed --- src/server/pullJobScheduler.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/server/pullJobScheduler.ts b/src/server/pullJobScheduler.ts index 8e6595833..7534df06d 100644 --- a/src/server/pullJobScheduler.ts +++ b/src/server/pullJobScheduler.ts @@ -11,7 +11,6 @@ import { Role, } from '@models'; import pubsub from './pubsub'; -import { CronJob } from 'cron'; // import * as CryptoJS from 'crypto-js'; import mongoose from 'mongoose'; import { getNextId, transformRecord } from '@utils/form'; @@ -96,8 +95,8 @@ const getUserRoleFiltersFromApp = (appName: string): any => { * * @param data array of data fetched from API * @param pullJob pull job configuration - * @param fromRoute tells if the insertion is done from pull-job or route * @param isEIOS is EIOS pulljob or not + * @param fromRoute tells if the insertion is done from pull-job or route */ export const insertRecords = async ( data: any[],