diff --git a/__tests__/models/pullJob.test.ts b/__tests__/models/pullJob.test.ts deleted file mode 100644 index 928f6e8e5..000000000 --- a/__tests__/models/pullJob.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { PullJob, Form, Channel, ApiConfiguration, Resource } from '@models'; -import { status } from '@const/enumTypes'; -import { faker } from '@faker-js/faker'; - -/** - * Test PullJob Model. - */ -describe('PullJob models tests', () => { - let pullJob: PullJob; - test('test PullJob model with correct data', async () => { - const apiConfiguration = await new ApiConfiguration({ - name: faker.internet.userName(), - }).save(); - const formName = faker.random.alpha(10); - - const resource = await new Resource({ - name: formName, - }).save(); - - const form = await new Form({ - name: formName, - graphQLTypeName: formName, - resource: resource._id, - }).save(); - const channel = await new Channel({ - title: faker.internet.userName(), - }).save(); - - for (let i = 0; i < 1; i++) { - const inputData = { - name: faker.word.adjective(), - status: status.active, - apiConfiguration: apiConfiguration._id, - url: faker.internet.url(), - path: faker.system.directoryPath(), - schedule: '* * * * *', - convertTo: form._id, - channel: channel._id, - }; - pullJob = await new PullJob(inputData).save(); - expect(pullJob._id).toBeDefined(); - } - }); - - test('test pullJob with duplicate name', async () => { - const duplicatePullJob = { - name: pullJob.name, - }; - expect(async () => - new PullJob(duplicatePullJob).save() - ).rejects.toThrowError( - 'E11000 duplicate key error collection: test.pulljobs index: name_1 dup key' - ); - }); - - test('test PullJob model with wrong status', async () => { - for (let i = 0; i < 1; i++) { - const inputData = { - name: faker.word.adjective(), - status: faker.word.adjective(), - }; - expect(async () => new PullJob(inputData).save()).rejects.toThrow(Error); - } - }); -}); diff --git a/__tests__/schema/query/pullJob.test.ts b/__tests__/schema/query/pullJob.test.ts deleted file mode 100644 index 6df36c6c9..000000000 --- a/__tests__/schema/query/pullJob.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { ApolloServer } from 'apollo-server-express'; -import schema from '../../../src/schema'; -import { SafeTestServer } from '../../server.setup'; -import { PullJob, Role } from '@models'; - -let server: ApolloServer; - -/** - * Test PullJob query. - */ -describe('PullJob query tests', () => { - const query = '{ pullJobs { totalCount, edges { node { id } } } }'; - - test('query with wrong user returns error', async () => { - server = await SafeTestServer.createApolloTestServer(schema, { - name: 'Wrong user', - roles: [], - }); - const result = await server.executeOperation({ query }); - expect(result.errors).toBeUndefined(); - expect(result).toHaveProperty(['data', 'pullJobs', 'totalCount']); - expect(result.data?.pullJobs.edges).toEqual([]); - expect(result.data?.pullJobs.totalCount).toEqual(0); - }); - - test('query with admin user returns expected number of pullJobs', async () => { - const count = await PullJob.countDocuments(); - const admin = await Role.findOne( - { title: 'admin' }, - 'id permissions' - ).populate({ - path: 'permissions', - model: 'Permission', - }); - server = await SafeTestServer.createApolloTestServer(schema, { - name: 'Admin user', - roles: [admin], - }); - const result = await server.executeOperation({ query }); - - expect(result.errors).toBeUndefined(); - expect(result).toHaveProperty(['data', 'pullJobs', 'totalCount']); - expect(result.data?.pullJobs.totalCount).toEqual(count); - }); -}); diff --git a/migrations/1664441608216-placeholders.ts b/migrations/1664441608216-placeholders.ts index d211ef075..a43aea590 100644 --- a/migrations/1664441608216-placeholders.ts +++ b/migrations/1664441608216-placeholders.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-loop-func */ import { startDatabaseForMigration } from '../src/utils/migrations/database.helper'; -import { Dashboard, Resource, PullJob, ReferenceData } from '../src/models'; +import { Dashboard, Resource, ReferenceData } from '../src/models'; import { Placeholder } from '../src/const/placeholders'; import { logger } from '../src/services/logger.service'; import get from 'lodash/get'; @@ -174,36 +174,6 @@ const migratePlaceholders = async () => { } await Promise.all(layoutPromises); - // === PULLJOB === - logger.info('Start pulljob update'); - const pullJobs = await PullJob.find({}); - const mappingPromises: Promise[] = []; - for (const pullJob of pullJobs) { - let modified = false; - for (const key in pullJob.mapping) { - if (pullJob.mapping[key].startsWith('$$')) { - pullJob.mapping[key] = '{{' + pullJob.mapping[key].slice(2) + '}}'; - modified = true; - } - } - if (modified) { - pullJob.markModified('mapping'); - mappingPromises.push( - pullJob.save().then( - () => { - logger.info('Pulljob "' + pullJob.name + '" mapping updated.'); - }, - (err) => { - logger.info( - 'Could not update pullJob "' + pullJob.name + '" mapping.\n' + err - ); - } - ) - ); - } - } - await Promise.all(mappingPromises); - // === REFERENCE DATA === logger.info('Start reference data update'); const referenceDatas = await ReferenceData.find({ diff --git a/src/index.ts b/src/index.ts index e1c3cd729..81dd6ed28 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,5 @@ import { SafeServer } from './server'; import mongoose from 'mongoose'; -import pullJobScheduler from './server/pullJobScheduler'; import customNotificationScheduler from './server/customNotificationScheduler'; import { startDatabase } from './server/database'; import config from 'config'; @@ -31,7 +30,6 @@ startDatabase(); mongoose.connection.once('open', () => { logger.log({ level: 'info', message: '📶 Connected to database' }); // subscriberSafe(); - pullJobScheduler(); customNotificationScheduler(); }); diff --git a/src/schema/mutation/addPullJob.mutation.ts b/src/schema/mutation/addPullJob.mutation.ts deleted file mode 100644 index c36411c2a..000000000 --- a/src/schema/mutation/addPullJob.mutation.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { - GraphQLError, - GraphQLID, - GraphQLList, - GraphQLNonNull, - GraphQLString, -} from 'graphql'; -import { PullJobType } from '../types'; -import { StatusType, status } from '@const/enumTypes'; -import { Channel, Form, PullJob } from '@models'; -import { StatusEnumType } from '@const/enumTypes'; -import GraphQLJSON from 'graphql-type-json'; -import { scheduleJob, unscheduleJob } from '../../server/pullJobScheduler'; -import { AppAbility } from '@security/defineUserAbility'; -import { logger } from '@services/logger.service'; -import { graphQLAuthCheck } from '@schema/shared'; -import { Types } from 'mongoose'; -import { Context } from '@server/apollo/context'; - -/** Arguments for the addPullJob mutation */ -type AddPullJobArgs = { - name: string; - status: StatusType; - apiConfiguration: string | Types.ObjectId; - url?: string; - path?: string; - schedule?: string; - convertTo?: string | Types.ObjectId; - mapping?: any; - uniqueIdentifiers?: string[]; - channel?: string | Types.ObjectId; -}; - -/** - * Creates a new pulljob. - * Throw an error if the user is not logged or authorized or if the form or channel aren't found. - */ -export default { - type: PullJobType, - args: { - name: { type: new GraphQLNonNull(GraphQLString) }, - status: { type: new GraphQLNonNull(StatusEnumType) }, - apiConfiguration: { type: new GraphQLNonNull(GraphQLID) }, - url: { type: GraphQLString }, - path: { type: GraphQLString }, - schedule: { type: GraphQLString }, - convertTo: { type: GraphQLID }, - mapping: { type: GraphQLJSON }, - uniqueIdentifiers: { type: new GraphQLList(GraphQLString) }, - channel: { type: GraphQLID }, - }, - async resolve(parent, args: AddPullJobArgs, context: Context) { - graphQLAuthCheck(context); - try { - const user = context.user; - - const ability: AppAbility = user.ability; - if (ability.can('create', 'PullJob')) { - if (args.convertTo) { - const form = await Form.findById(args.convertTo); - if (!form) - throw new GraphQLError( - context.i18next.t('common.errors.dataNotFound') - ); - } - - if (args.channel) { - const filters = { - _id: args.channel, - }; - const channel = await Channel.findOne(filters); - if (!channel) - throw new GraphQLError( - context.i18next.t('common.errors.dataNotFound') - ); - } - - try { - // Create a new PullJob - const pullJob = new PullJob({ - name: args.name, - status: args.status, - apiConfiguration: args.apiConfiguration, - url: args.url, - path: args.path, - schedule: args.schedule, - convertTo: args.convertTo, - mapping: args.mapping, - uniqueIdentifiers: args.uniqueIdentifiers, - channel: args.channel, - }); - await pullJob.save(); - - // If the pullJob is active, schedule it immediately - if (args.status === status.active) { - const fullPullJob = await PullJob.findById(pullJob.id).populate({ - path: 'apiConfiguration', - model: 'ApiConfiguration', - }); - scheduleJob(fullPullJob); - } else { - unscheduleJob(pullJob); - } - return pullJob; - } catch (err) { - logger.error(err.message); - throw new GraphQLError(err.message); - } - } else { - throw new GraphQLError( - context.i18next.t('common.errors.permissionNotGranted') - ); - } - } catch (err) { - logger.error(err.message, { stack: err.stack }); - if (err instanceof GraphQLError) { - throw new GraphQLError(err.message); - } - throw new GraphQLError( - context.i18next.t('common.errors.internalServerError') - ); - } - }, -}; diff --git a/src/schema/mutation/deletePullJob.mutation.ts b/src/schema/mutation/deletePullJob.mutation.ts deleted file mode 100644 index dce9f9190..000000000 --- a/src/schema/mutation/deletePullJob.mutation.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { GraphQLError, GraphQLID, GraphQLNonNull } from 'graphql'; -import { PullJobType } from '../types'; -import { PullJob } from '@models'; -import { AppAbility } from '@security/defineUserAbility'; -import { unscheduleJob } from '../../server/pullJobScheduler'; -import { logger } from '@services/logger.service'; -import { accessibleBy } from '@casl/mongoose'; -import { graphQLAuthCheck } from '@schema/shared'; -import { Types } from 'mongoose'; -import { Context } from '@server/apollo/context'; - -/** Arguments for the deletePullJob mutation */ -type DeletePullJobArgs = { - id: string | Types.ObjectId; -}; - -/** - * Delete a pullJob - */ -export default { - type: PullJobType, - args: { - id: { type: new GraphQLNonNull(GraphQLID) }, - }, - async resolve(parent, args: DeletePullJobArgs, context: Context) { - graphQLAuthCheck(context); - try { - const user = context.user; - const ability: AppAbility = user.ability; - - const filters = PullJob.find(accessibleBy(ability, 'delete').PullJob) - .where({ _id: args.id }) - .getFilter(); - const pullJob = await PullJob.findOneAndDelete(filters); - if (!pullJob) - throw new GraphQLError( - context.i18next.t('common.errors.permissionNotGranted') - ); - - unscheduleJob(pullJob); - return pullJob; - } catch (err) { - logger.error(err.message, { stack: err.stack }); - if (err instanceof GraphQLError) { - throw new GraphQLError(err.message); - } - throw new GraphQLError( - context.i18next.t('common.errors.internalServerError') - ); - } - }, -}; diff --git a/src/schema/mutation/editPullJob.mutation.ts b/src/schema/mutation/editPullJob.mutation.ts deleted file mode 100644 index 4cf2a6d0c..000000000 --- a/src/schema/mutation/editPullJob.mutation.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { - GraphQLError, - GraphQLID, - GraphQLList, - GraphQLNonNull, - GraphQLString, -} from 'graphql'; -import { PullJobType } from '../types'; -import { StatusType, status } from '@const/enumTypes'; -import { Channel, Form, PullJob } from '@models'; -import { AppAbility } from '@security/defineUserAbility'; -import { StatusEnumType } from '@const/enumTypes'; -import GraphQLJSON from 'graphql-type-json'; -import { scheduleJob, unscheduleJob } from '../../server/pullJobScheduler'; -import { logger } from '@services/logger.service'; -import { accessibleBy } from '@casl/mongoose'; -import { graphQLAuthCheck } from '@schema/shared'; -import { Types } from 'mongoose'; -import { Context } from '@server/apollo/context'; - -/** Arguments for the editPullJob mutation */ -type EditPullJobArgs = { - id: string | Types.ObjectId; - name?: string; - status?: StatusType; - apiConfiguration?: string | Types.ObjectId; - url?: string; - path?: string; - schedule?: string; - convertTo?: string | Types.ObjectId; - mapping?: any; - uniqueIdentifiers?: string[]; - channel?: string | Types.ObjectId; -}; - -/** - * Edit an existing pullJob if authorized. - */ -export default { - type: PullJobType, - args: { - id: { type: new GraphQLNonNull(GraphQLID) }, - name: { type: GraphQLString }, - status: { type: StatusEnumType }, - apiConfiguration: { type: GraphQLID }, - url: { type: GraphQLString }, - path: { type: GraphQLString }, - schedule: { type: GraphQLString }, - convertTo: { type: GraphQLID }, - mapping: { type: GraphQLJSON }, - uniqueIdentifiers: { type: new GraphQLList(GraphQLString) }, - channel: { type: GraphQLID }, - }, - async resolve(parent, args: EditPullJobArgs, context: Context) { - graphQLAuthCheck(context); - try { - const user = context.user; - const ability: AppAbility = user.ability; - - if (args.convertTo) { - const form = await Form.findById(args.convertTo); - if (!form) - throw new GraphQLError( - context.i18next.t('common.errors.dataNotFound') - ); - } - - if (args.channel) { - const filters = { - _id: args.channel, - }; - const channel = await Channel.findOne(filters); - if (!channel) - throw new GraphQLError( - context.i18next.t('common.errors.dataNotFound') - ); - } - - const update = {}; - Object.assign( - update, - args.name && { name: args.name }, - args.status && { status: args.status }, - args.apiConfiguration && { apiConfiguration: args.apiConfiguration }, - args.url && { url: args.url }, - args.path && { path: args.path }, - args.schedule && { schedule: args.schedule }, - args.convertTo && { convertTo: args.convertTo }, - args.mapping && { mapping: args.mapping }, - args.uniqueIdentifiers && { uniqueIdentifiers: args.uniqueIdentifiers }, - args.channel && { channel: args.channel } - ); - const filters = PullJob.find(accessibleBy(ability, 'update').PullJob) - .where({ _id: args.id }) - .getFilter(); - try { - const pullJob = await PullJob.findOneAndUpdate(filters, update, { - new: true, - runValidators: true, - }).populate({ - path: 'apiConfiguration', - model: 'ApiConfiguration', - }); - if (!pullJob) - throw new GraphQLError( - context.i18next.t('common.errors.dataNotFound') - ); - if (pullJob.status === status.active) { - scheduleJob(pullJob); - } else { - unscheduleJob(pullJob); - } - return pullJob; - } catch (err) { - logger.error(err.message); - throw new GraphQLError(err.message); - } - } catch (err) { - logger.error(err.message, { stack: err.stack }); - if (err instanceof GraphQLError) { - throw new GraphQLError(err.message); - } - throw new GraphQLError( - context.i18next.t('common.errors.internalServerError') - ); - } - }, -}; diff --git a/src/schema/mutation/index.ts b/src/schema/mutation/index.ts index 5814f2446..33602be09 100644 --- a/src/schema/mutation/index.ts +++ b/src/schema/mutation/index.ts @@ -52,9 +52,6 @@ import editUserProfile from './editUserProfile.mutation'; import addApiConfiguration from './addApiConfiguration.mutation'; import editApiConfiguration from './editApiConfiguration.mutation'; import deleteApiConfiguration from './deleteApiConfiguration.mutation'; -import addPullJob from './addPullJob.mutation'; -import editPullJob from './editPullJob.mutation'; -import deletePullJob from './deletePullJob.mutation'; import toggleApplicationLock from './toggleApplicationLock.mutation'; import addUsers from './addUsers.mutation'; import addLayout from './addLayout.mutation'; @@ -104,7 +101,6 @@ const Mutation = new GraphQLObjectType({ addPage, addPositionAttribute, addPositionAttributeCategory, - addPullJob, addRecord, addReferenceData, addRole, @@ -124,7 +120,6 @@ const Mutation = new GraphQLObjectType({ deleteLayout, deletePage, deletePositionAttributeCategory, - deletePullJob, deleteRecord, deleteRecords, deleteReferenceData, @@ -147,7 +142,6 @@ const Mutation = new GraphQLObjectType({ editPage, editPageContext, editPositionAttributeCategory, - editPullJob, editRecord, editRecords, editReferenceData, diff --git a/src/schema/query/index.ts b/src/schema/query/index.ts index 74920da62..bff4a4246 100644 --- a/src/schema/query/index.ts +++ b/src/schema/query/index.ts @@ -27,7 +27,6 @@ import channels from './channels.query'; import positionAttributes from './positionAttributes.query'; import apiConfiguration from './apiConfiguration.query'; import apiConfigurations from './apiConfigurations.query'; -import pullJobs from './pullJobs.query'; import referenceData from './referenceData.query'; import referenceDatas from './referenceDatas.query'; import recordHistory from './recordHistory.query'; @@ -60,7 +59,6 @@ const Query = new GraphQLObjectType({ page, pages, permissions, - pullJobs, record, records, recordsAggregation, diff --git a/src/schema/query/pullJobs.query.ts b/src/schema/query/pullJobs.query.ts deleted file mode 100644 index e6050b4e4..000000000 --- a/src/schema/query/pullJobs.query.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { GraphQLError, GraphQLID, GraphQLInt } from 'graphql'; -import { PullJobConnectionType, encodeCursor, decodeCursor } from '../types'; -import { PullJob } from '@models'; -import { AppAbility } from '@security/defineUserAbility'; -import { logger } from '@services/logger.service'; -import checkPageSize from '@utils/schema/errors/checkPageSize.util'; -import { accessibleBy } from '@casl/mongoose'; -import { graphQLAuthCheck } from '@schema/shared'; -import { Context } from '@server/apollo/context'; - -/** Default page size */ -const DEFAULT_FIRST = 10; - -/** Arguments for the pullJobs query */ -type PullJobsArgs = { - first?: number; - afterCursor?: string; -}; - -/** - * Return all pull jobs available for the logged user. - * Throw GraphQL error if not logged. - */ -export default { - type: PullJobConnectionType, - args: { - first: { type: GraphQLInt }, - afterCursor: { type: GraphQLID }, - }, - async resolve(parent, args: PullJobsArgs, context: Context) { - graphQLAuthCheck(context); - // Make sure that the page size is not too important - const first = args.first || DEFAULT_FIRST; - checkPageSize(first); - try { - const ability: AppAbility = context.user.ability; - const abilityFilters = PullJob.find( - accessibleBy(ability, 'read').PullJob - ).getFilter(); - const filters: any[] = [abilityFilters]; - - const afterCursor = args.afterCursor; - const cursorFilters = afterCursor - ? { - _id: { - $gt: decodeCursor(afterCursor), - }, - } - : {}; - - let items: any[] = await PullJob.find({ - $and: [cursorFilters, ...filters], - }) - .sort({ _id: 1 }) - .limit(first + 1); - - const hasNextPage = items.length > first; - if (hasNextPage) { - items = items.slice(0, items.length - 1); - } - const edges = items.map((r) => ({ - cursor: encodeCursor(r.id.toString()), - node: r, - })); - return { - pageInfo: { - hasNextPage, - startCursor: edges.length > 0 ? edges[0].cursor : null, - endCursor: edges.length > 0 ? edges[edges.length - 1].cursor : null, - }, - edges, - totalCount: await PullJob.countDocuments({ $and: filters }), - }; - } catch (err) { - logger.error(err.message, { stack: err.stack }); - if (err instanceof GraphQLError) { - throw new GraphQLError(err.message); - } - throw new GraphQLError( - context.i18next.t('common.errors.internalServerError') - ); - } - }, -}; diff --git a/src/schema/types/application.type.ts b/src/schema/types/application.type.ts index b3ea87232..8b1e77915 100644 --- a/src/schema/types/application.type.ts +++ b/src/schema/types/application.type.ts @@ -14,7 +14,6 @@ import { Channel, Application, PositionAttributeCategory, - PullJob, } from '@models'; import mongoose from 'mongoose'; import { @@ -23,7 +22,6 @@ import { RoleType, AccessType, PositionAttributeCategoryType, - PullJobType, TemplateType, DistributionListType, encodeCursor, @@ -448,17 +446,6 @@ export const ApplicationType = new GraphQLObjectType({ }, }, subscriptions: { type: new GraphQLList(SubscriptionType) }, - pullJobs: { - type: new GraphQLList(PullJobType), - async resolve(parent, args, context) { - const ability: AppAbility = context.user.ability; - const pullJobs = PullJob.find({ - ...accessibleBy(ability, 'read').PullJob, - _id: { $in: parent.pullJobs }, - }); - return pullJobs; - }, - }, permissions: { type: AccessType, resolve(parent, args, context) { diff --git a/src/schema/types/index.ts b/src/schema/types/index.ts index e87cccf86..5444273e0 100644 --- a/src/schema/types/index.ts +++ b/src/schema/types/index.ts @@ -9,7 +9,6 @@ export * from './page.type'; export * from './permission.type'; export * from './positionAttribute.type'; export * from './positionAttributeCategory.type'; -export * from './pullJob.type'; export * from './record.type'; export * from './referenceData.type'; export * from './resource.type'; diff --git a/src/schema/types/pullJob.type.ts b/src/schema/types/pullJob.type.ts deleted file mode 100644 index 921914a51..000000000 --- a/src/schema/types/pullJob.type.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { - GraphQLObjectType, - GraphQLID, - GraphQLString, - GraphQLList, -} from 'graphql'; -import GraphQLJSON from 'graphql-type-json'; -import { ApiConfiguration, Form, Channel } from '@models'; -import { StatusEnumType } from '@const/enumTypes'; -import { AppAbility } from '@security/defineUserAbility'; -import { ApiConfigurationType } from './apiConfiguration.type'; -import { ChannelType } from './channel.type'; -import { FormType } from './form.type'; -import { Connection } from './pagination.type'; -import { accessibleBy } from '@casl/mongoose'; - -/** GraphQL pull job type definition */ -export const PullJobType = new GraphQLObjectType({ - name: 'PullJob', - fields: () => ({ - id: { type: GraphQLID }, - name: { type: GraphQLString }, - status: { type: StatusEnumType }, - apiConfiguration: { - type: ApiConfigurationType, - async resolve(parent, args, context) { - const ability: AppAbility = context.user.ability; - const apiConfig = await ApiConfiguration.findOne({ - _id: parent.apiConfiguration, - ...accessibleBy(ability, 'read').ApiConfiguration, - }); - return apiConfig; - }, - }, - url: { type: GraphQLString }, - path: { type: GraphQLString }, - schedule: { type: GraphQLString }, - convertTo: { - type: FormType, - async resolve(parent, args, context) { - const ability: AppAbility = context.user.ability; - const form = await Form.findOne({ - _id: parent.convertTo, - ...accessibleBy(ability, 'read').Form, - }); - return form; - }, - }, - mapping: { type: GraphQLJSON }, - uniqueIdentifiers: { type: new GraphQLList(GraphQLString) }, - channel: { - type: ChannelType, - async resolve(parent, args, context) { - const ability: AppAbility = context.user.ability; - const channel = await Channel.findOne({ - _id: parent.channel, - ...accessibleBy(ability, 'read').Channel, - }); - return channel; - }, - }, - }), -}); - -/** GraphQL pull job connection type definition */ -export const PullJobConnectionType = Connection(PullJobType); diff --git a/src/security/defineUserAbility.ts b/src/security/defineUserAbility.ts index 3e4072cfb..c389337c5 100644 --- a/src/security/defineUserAbility.ts +++ b/src/security/defineUserAbility.ts @@ -27,7 +27,6 @@ import { User, Version, Workflow, - PullJob, ReferenceData, Group, Template, @@ -59,7 +58,6 @@ type Models = | Notification | Page | Permission - | PullJob | Record | ReferenceData | Resource @@ -318,12 +316,12 @@ export default function defineUserAbility(user: User | Client): AppAbility { }); /* === - Creation / Access / Edition / Deletion of API configurations, PullJobs and ReferenceData + Creation / Access / Edition / Deletion of API configurations and ReferenceData === */ if (userGlobalPermissions.includes(permissions.canManageApiConfigurations)) { can( ['create', 'read', 'update', 'delete'], - ['ApiConfiguration', 'PullJob', 'ReferenceData'] + ['ApiConfiguration', 'ReferenceData'] ); } else { // can('read', 'ApiConfiguration', filters('canSee', user)); diff --git a/src/server/pullJobScheduler.ts b/src/server/pullJobScheduler.ts index 30710da32..7534df06d 100644 --- a/src/server/pullJobScheduler.ts +++ b/src/server/pullJobScheduler.ts @@ -1,10 +1,8 @@ -import { authType } from '@const/enumTypes'; import { BASE_PLACEHOLDER_REGEX, extractStringFromBrackets, } from '../const/placeholders'; import { - ApiConfiguration, Form, Notification, PullJob, @@ -13,20 +11,12 @@ import { Role, } from '@models'; import pubsub from './pubsub'; -import { CronJob } from 'cron'; // import * as CryptoJS from 'crypto-js'; import mongoose from 'mongoose'; -import { getToken } from '@utils/proxy'; import { getNextId, transformRecord } from '@utils/form'; import { logger } from '../services/logger.service'; -import * as cronValidator from 'cron-validator'; -import get from 'lodash/get'; -import axios from 'axios'; import { ownershipMappingJSON } from './EIOSOwnernshipMapping'; -/** A map with the task ids as keys and the scheduled tasks as values */ -const taskMap: Record = {}; - /** Record's default fields */ const DEFAULT_FIELDS = ['createdBy']; @@ -42,198 +32,6 @@ const EIOS_APP_NAMES: string[] = [ ), ]; -/** - * Global function called on server start to initialize all the pullJobs. - */ -const pullJobScheduler = async () => { - const pullJobs = await PullJob.find({ status: 'active' }).populate({ - path: 'apiConfiguration', - model: 'ApiConfiguration', - }); - - for (const pullJob of pullJobs) { - // eslint-disable-next-line @typescript-eslint/no-use-before-define - scheduleJob(pullJob); - } -}; - -export default pullJobScheduler; - -/** - * Schedule or re-schedule a pullJob. - * - * @param pullJob pull job to schedule - */ -export const scheduleJob = (pullJob: PullJob) => { - try { - const task = taskMap[pullJob.id]; - if (task) { - task.stop(); - } - const schedule = get(pullJob, 'schedule', ''); - if (cronValidator.isValidCron(schedule)) { - taskMap[pullJob.id] = new CronJob( - pullJob.schedule, - async () => { - logger.info('📥 Starting a pull from job ' + pullJob.name); - const apiConfiguration: ApiConfiguration = pullJob.apiConfiguration; - try { - if (apiConfiguration.authType === authType.serviceToService) { - // Decrypt settings - // const settings: { - // authTargetUrl: string; - // apiClientID: string; - // safeSecret: string; - // scope: string; - // } = JSON.parse( - // CryptoJS.AES.decrypt( - // apiConfiguration.settings, - // config.get('encryption.key') - // ).toString(CryptoJS.enc.Utf8) - // ); - - // Get auth token and start pull Logic - const token: string = await getToken(apiConfiguration); - // eslint-disable-next-line @typescript-eslint/no-use-before-define - fetchRecordsServiceToService(pullJob, token); - } - if (apiConfiguration.authType === authType.public) { - // eslint-disable-next-line @typescript-eslint/no-use-before-define - fetchRecordsPublic(pullJob); - } - if (apiConfiguration.authType === authType.authorizationCode) { - throw new Error( - 'Unsupported Api configuration with Authorization Code authentication.' - ); - } - } catch (err) { - logger.error(err.message, { stack: err.stack }); - } - }, - null, - true - ); - logger.info('📅 Scheduled job ' + pullJob.name); - } else { - throw new Error(`[${pullJob.name}] Invalid schedule: ${schedule}`); - } - } catch (err) { - logger.error(err.message); - } -}; - -/** - * Unschedule an existing pullJob from its id. - * - * @param pullJob pull job to unschedule - */ -export const unscheduleJob = (pullJob: PullJob): void => { - const task = taskMap[pullJob.id]; - if (task) { - task.stop(); - logger.info( - `📆 Unscheduled job ${pullJob.name ? pullJob.name : pullJob.id}` - ); - } -}; - -/** - * Fetch records using the hardcoded workflow for service-to-service API type (EIOS). - * - * @param pullJob pull job configuration to use - * @param token authentication token - */ -const fetchRecordsServiceToService = ( - pullJob: PullJob, - token: string -): void => { - const apiConfiguration: ApiConfiguration = pullJob.apiConfiguration; - // Hard coded for EIOS due to specific behavior - const EIOS_ORIGIN = 'https://portal.who.int/eios/'; - // === HARD CODED ENDPOINTS === - const headers: any = { - Authorization: 'Bearer ' + token, - }; - // Hardcoded specific behavior for EIOS - if (apiConfiguration.endpoint.startsWith(EIOS_ORIGIN)) { - // === HARD CODED ENDPOINTS === - const boardsUrl = 'GetBoards?tags=signal+app'; - const articlesUrl = 'GetPinnedArticles'; - axios({ - url: apiConfiguration.endpoint + boardsUrl, - method: 'get', - headers, - }) - .then(({ data }) => { - if (data && data.result) { - const boardIds = data.result.map((x) => x.id); - axios({ - url: `${apiConfiguration.endpoint}${articlesUrl}?boardIds=${boardIds}`, - method: 'get', - headers, - }) - .then(({ data: data2 }) => { - if (data2 && data2.result) { - // eslint-disable-next-line @typescript-eslint/no-use-before-define - insertRecords(data2.result, pullJob, true, false); - } - }) - .catch((err) => { - logger.error( - `Job ${pullJob.name} : Failed to get pinned articles : ${err}` - ); - }); - } - }) - .catch((err) => { - logger.error( - `Job ${pullJob.name} : Failed to get signal app boards : ${err}` - ); - }); - } else { - // Generic case - axios({ - url: apiConfiguration.endpoint + pullJob.url, - method: 'get', - headers, - }) - .then(({ data }) => { - const records = pullJob.path ? get(data, pullJob.path) : data; - if (records) { - // eslint-disable-next-line @typescript-eslint/no-use-before-define - insertRecords(records, pullJob, false, false); - } - }) - .catch((err) => { - logger.error(`Job ${pullJob.name} : Failed to fetch data : ${err}`); - }); - } -}; - -/** - * Fetch records using the generic workflow for public endpoints. - * - * @param pullJob pull job to use - */ -const fetchRecordsPublic = (pullJob: PullJob): void => { - const apiConfiguration: ApiConfiguration = pullJob.apiConfiguration; - logger.info(`Execute pull job operation: ${pullJob.name}`); - axios({ - url: apiConfiguration.endpoint + pullJob.url, - method: 'get', - }) - .then(({ data }) => { - const records = pullJob.path ? get(data, pullJob.path) : data; - if (records) { - // eslint-disable-next-line @typescript-eslint/no-use-before-define - insertRecords(records, pullJob, false, false); - } - }) - .catch((err) => { - logger.error(`Job ${pullJob.name} : Failed to fetch data : ${err}`); - }); -}; - /** * Access property of passed object including nested properties and map properties on array if needed. * @@ -297,8 +95,8 @@ const getUserRoleFiltersFromApp = (appName: string): any => { * * @param data array of data fetched from API * @param pullJob pull job configuration - * @param fromRoute tells if the insertion is done from pull-job or route * @param isEIOS is EIOS pulljob or not + * @param fromRoute tells if the insertion is done from pull-job or route */ export const insertRecords = async ( data: any[],