From 49134d1823713ff62b4275b0e6d75d478a2bfd67 Mon Sep 17 00:00:00 2001 From: Paula Stachova Date: Fri, 31 Jan 2025 14:30:19 +0100 Subject: [PATCH 01/13] feat: internal to Standard JSON Schema conversion COMPASS-8700 --- src/schema-convertors/internalToStandard.ts | 320 +++++++++++++++++++- 1 file changed, 314 insertions(+), 6 deletions(-) diff --git a/src/schema-convertors/internalToStandard.ts b/src/schema-convertors/internalToStandard.ts index 3d58125..631bab2 100644 --- a/src/schema-convertors/internalToStandard.ts +++ b/src/schema-convertors/internalToStandard.ts @@ -1,12 +1,320 @@ -import { InternalSchema } from '..'; +import { ArraySchemaType, DocumentSchemaType, Schema as InternalSchema, SchemaType } from '../schema-analyzer'; import { StandardJSONSchema } from '../types'; -export default function internalSchemaToStandard( - /* eslint @typescript-eslint/no-unused-vars: 0 */ +const InternalTypeToStandardTypeMap: Record< + SchemaType['name'] | 'Double' | 'BSONSymbol', + string | { $ref: string } +> = { + Double: { $ref: '#/$defs/Double' }, + Number: { $ref: '#/$defs/Double' }, + String: 'string', + Document: 'object', + Array: 'array', + Binary: { $ref: '#/$defs/Binary' }, + Undefined: { $ref: '#/$defs/Undefined' }, + ObjectId: { $ref: '#/$defs/ObjectId' }, + Boolean: 'boolean', + Date: { $ref: '#/$defs/Date' }, + Null: 'null', + RegExp: { $ref: '#/$defs/RegExp' }, + BSONRegExp: { $ref: '#/$defs/RegExp' }, + DBRef: { $ref: '#/$defs/DBRef' }, + DBPointer: { $ref: '#/$defs/DBPointer' }, + BSONSymbol: { $ref: '#/$defs/BSONSymbol' }, + Symbol: { $ref: '#/$defs/BSONSymbol' }, + Code: { $ref: '#/$defs/Code' }, + Int32: 'integer', + Timestamp: { $ref: '#/$defs/Timestamp' }, + Long: 'integer', + Decimal128: { $ref: '#/$defs/Decimal' }, + MinKey: { $ref: '#/$defs/MinKey' }, + MaxKey: { $ref: '#/$defs/MaxKey' } +}; + +const RELAXED_EJSON_DEFINITIONS = Object.freeze({ + ObjectId: { + type: 'object', + properties: { + $oid: { + type: 'string', + pattern: '^[0-9a-fA-F]{24}$' + } + }, + required: ['$oid'], + additionalProperties: false + }, + BSONSymbol: { + type: 'object', + properties: { + $symbol: { + type: 'string' + } + }, + required: ['$symbol'], + additionalProperties: false + }, + Double: { + oneOf: [ + { type: 'number' }, + { + enum: ['Infinity', '-Infinity', 'NaN'] + } + ] + }, + Decimal128: { + type: 'object', + properties: { + $numberDecimal: { + type: 'string' + } + }, + required: ['$numberDecimal'], + additionalProperties: false + }, + Binary: { + type: 'object', + properties: { + $binary: { + type: 'object', + properties: { + base64: { + type: 'string' + }, + subType: { + type: 'string', + pattern: '^[0-9a-fA-F]{1,2}$' // BSON binary type as a one- or two-character hex string + } + }, + required: ['base64', 'subType'], + additionalProperties: false + } + }, + required: ['$binary'], + additionalProperties: false + }, + Code: { + type: 'object', + properties: { + $code: { + type: 'string' + } + }, + required: ['$code'], + additionalProperties: false + }, + CodeWScope: { + type: 'object', + properties: { + $code: { + type: 'string' + }, + $scope: { + type: 'object' // TODO: object is ejson object hmm + } + }, + required: ['$code', '$scope'], + additionalProperties: false + }, + Timestamp: { + type: 'object', + properties: { + $timestamp: { + type: 'object', + properties: { + t: { + type: 'integer', + minimum: 0 + }, + i: { + type: 'integer', + minimum: 0 + } + }, + required: ['t', 'i'], + additionalProperties: false + } + }, + required: ['$timestamp'], + additionalProperties: false + }, + RegExp: { + type: 'object', + properties: { + $regularExpression: { + type: 'object', + properties: { + pattern: { + type: 'string' + }, + options: { + type: 'string', + pattern: '^[gimuy]*$' + } + }, + required: ['pattern'], + additionalProperties: false + } + }, + required: ['$regularExpression'], + additionalProperties: false + }, + DBPointer: { + type: 'object', + properties: { + $dbPointer: { + type: 'object', + properties: { + $ref: { + type: 'string' + }, + $id: { + $ref: '#/$defs/Decimal' + } + }, + required: ['$ref', '$id'], + additionalProperties: false + } + }, + required: ['$dbPointer'], + additionalProperties: false + }, + Date: { + type: 'object', + properties: { + $date: { + type: 'string', + format: 'date-time' + } + }, + required: ['$date'], + additionalProperties: false + }, + DBRef: { + type: 'object', + properties: { + $ref: { + type: 'string' + }, + $id: {}, + $db: { + type: 'string' + } + }, + required: ['$ref', '$id'], + additionalProperties: true + }, + MinKey: { + type: 'object', + properties: { + $minKey: { + type: 'integer', + const: 1 + } + }, + required: ['$minKey'], + additionalProperties: false + }, + MaxKey: { + type: 'object', + properties: { + $maxKey: { + type: 'integer', + const: 1 + } + }, + required: ['$maxKey'], + additionalProperties: false + }, + Undefined: { + type: 'object', + properties: { + $undefined: { + type: 'boolean', + const: true + } + }, + required: ['$undefined'], + additionalProperties: false + } +}); + +const convertInternalType = (type: string) => { + const bsonType = InternalTypeToStandardTypeMap[type]; + if (!bsonType) throw new Error(`Encountered unknown type: ${type}`); + return bsonType; +}; + +async function allowAbort(signal?: AbortSignal) { + return new Promise((resolve, reject) => + setTimeout(() => { + if (signal?.aborted) return reject(signal?.reason || new Error('Operation aborted')); + resolve(); + }) + ); +} + +async function parseType(type: SchemaType, signal?: AbortSignal): Promise { + await allowAbort(signal); + const schema: StandardJSONSchema = { + bsonType: convertInternalType(type.bsonType) + }; + switch (type.bsonType) { + case 'Array': + schema.items = await parseTypes((type as ArraySchemaType).types); + break; + case 'Document': + Object.assign(schema, + await parseFields((type as DocumentSchemaType).fields, signal) + ); + break; + } + + return schema; +} + +async function parseTypes(types: SchemaType[], signal?: AbortSignal): Promise { + await allowAbort(signal); + const definedTypes = types.filter(type => type.bsonType.toLowerCase() !== 'undefined'); + const isSingleType = definedTypes.length === 1; + if (isSingleType) { + return parseType(definedTypes[0], signal); + } + const parsedTypes = await Promise.all(definedTypes.map(type => parseType(type, signal))); + if (definedTypes.some(type => ['Document', 'Array'].includes(type.bsonType))) { + return { + anyOf: parsedTypes + }; + } + return { + bsonType: definedTypes.map((type) => convertInternalType(type.bsonType)) + }; +} + +async function parseFields(fields: DocumentSchemaType['fields'], signal?: AbortSignal): Promise<{ + required: StandardJSONSchema['required'], + properties: StandardJSONSchema['properties'], +}> { + const required = []; + const properties: StandardJSONSchema['properties'] = {}; + for (const field of fields) { + if (field.probability === 1) required.push(field.name); + properties[field.name] = await parseTypes(field.types, signal); + } + + return { required, properties }; +} + +export default async function internalSchemaToMongodb( internalSchema: InternalSchema, options: { signal?: AbortSignal -}): Promise { - // TODO: COMPASS-8700 - return Promise.resolve({} as StandardJSONSchema); +} = {}): Promise { + const { required, properties } = await parseFields(internalSchema.fields, options.signal); + const schema: StandardJSONSchema = { + bsonType: 'object', + required, + properties, + $defs: RELAXED_EJSON_DEFINITIONS + }; + return schema; } From 40ac8ff93ca5d8dd6b0c64ee938ac04a61c258e7 Mon Sep 17 00:00:00 2001 From: Paula Stachova Date: Mon, 3 Feb 2025 14:35:13 +0100 Subject: [PATCH 02/13] wip --- .../internalToStandard.test.ts | 1775 +++++++++++++++++ src/schema-convertors/internalToStandard.ts | 48 +- 2 files changed, 1804 insertions(+), 19 deletions(-) create mode 100644 src/schema-convertors/internalToStandard.test.ts diff --git a/src/schema-convertors/internalToStandard.test.ts b/src/schema-convertors/internalToStandard.test.ts new file mode 100644 index 0000000..99acba4 --- /dev/null +++ b/src/schema-convertors/internalToStandard.test.ts @@ -0,0 +1,1775 @@ +import assert from 'assert'; +import internalSchemaToStandard, { RELAXED_EJSON_DEFINITIONS } from './internalToStandard'; + +describe('internalSchemaToStandard', async function() { + describe('Converts: ', async function() { + it.only('all the types', async function() { + const internal = { + count: 1, + fields: [ + { + name: '_id', + path: [ + '_id' + ], + count: 1, + type: 'ObjectId', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'ObjectId', + path: [ + '_id' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + '642d766b7300158b1f22e972' + ], + bsonType: 'ObjectId' + } + ] + }, + { + name: 'array', + path: [ + 'array' + ], + count: 1, + type: 'Array', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: [ + 'array' + ], + count: 1, + probability: 0.8, + bsonType: 'Array', + types: [ + { + name: 'Number', + path: [ + 'array' + ], + count: 3, + probability: 0.8, + unique: 3, + hasDuplicates: false, + values: [ + 1, + 2, + 3 + ], + bsonType: 'Number' + } + ], + totalCount: 3, + lengths: [ + 3 + ], + averageLength: 3 + } + ] + }, + { + name: 'binaries', + path: [ + 'binaries' + ], + count: 1, + type: 'Document', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Document', + path: [ + 'binaries' + ], + count: 1, + probability: 0.8, + bsonType: 'Document', + fields: [ + { + name: 'binaryOld', + path: [ + 'binaries', + 'binaryOld' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'binaryOld' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + '//8=' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'compressedTimeSeries', + path: [ + 'binaries', + 'compressedTimeSeries' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'compressedTimeSeries' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'c//SZESzTGmQ6OfR38A11A==' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'custom', + path: [ + 'binaries', + 'custom' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'custom' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + '//8=' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'encrypted', + path: [ + 'binaries', + 'encrypted' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'encrypted' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'c//SZESzTGmQ6OfR38A11A==' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'functionData', + path: [ + 'binaries', + 'functionData' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'functionData' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + '//8=' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'generic', + path: [ + 'binaries', + 'generic' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'generic' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'AQID' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'md5', + path: [ + 'binaries', + 'md5' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'md5' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'c//SZESzTGmQ6OfR38A11A==' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'uuid', + path: [ + 'binaries', + 'uuid' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'uuid' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'uuidOld', + path: [ + 'binaries', + 'uuidOld' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'uuidOld' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'c//SZESzTGmQ6OfR38A11A==' + ], + bsonType: 'Binary' + } + ] + } + ] + } + ] + }, + { + name: 'binData', + path: [ + 'binData' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binData' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'AQID' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'boolean', + path: [ + 'boolean' + ], + count: 1, + type: 'Boolean', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Boolean', + path: [ + 'boolean' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + true + ], + bsonType: 'Boolean' + } + ] + }, + { + name: 'date', + path: [ + 'date' + ], + count: 1, + type: 'Date', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Date', + path: [ + 'date' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + '2023-04-05T13:25:08.445Z' + ], + bsonType: 'Date' + } + ] + }, + { + name: 'dbRef', + path: [ + 'dbRef' + ], + count: 1, + type: 'DBRef', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'DBRef', + path: [ + 'dbRef' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + $ref: 'namespace', + $id: '642d76b4b7ebfab15d3c4a78' + } + ], + bsonType: 'DBRef' + } + ] + }, + { + name: 'decimal', + path: [ + 'decimal' + ], + count: 1, + type: 'Decimal128', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Decimal128', + path: [ + 'decimal' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + $numberDecimal: '5.477284286264328586719275128128001E-4088' + } + ], + bsonType: 'Decimal128' + } + ] + }, + { + name: 'double', + path: [ + 'double' + ], + count: 1, + type: 'Double', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Double', + path: [ + 'double' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 1.2 + ], + bsonType: 'Double' + } + ] + }, + { + name: 'int', + path: [ + 'int' + ], + count: 1, + type: 'Int32', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Int32', + path: [ + 'int' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 12345 + ], + bsonType: 'Int32' + } + ] + }, + { + name: 'javascript', + path: [ + 'javascript' + ], + count: 1, + type: 'Code', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Code', + path: [ + 'javascript' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + code: 'function() {}' + } + ], + bsonType: 'Code' + } + ] + }, + { + name: 'javascriptWithScope', + path: [ + 'javascriptWithScope' + ], + count: 1, + type: 'Code', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Code', + path: [ + 'javascriptWithScope' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + code: 'function() {}', + scope: { + foo: 1, + bar: 'a' + } + } + ], + bsonType: 'Code' + } + ] + }, + { + name: 'long', + path: [ + 'long' + ], + count: 1, + type: 'Long', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Long', + path: [ + 'long' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + low: -1395630315, + high: 28744523, + unsigned: false + } + ], + bsonType: 'Long' + } + ] + }, + { + name: 'maxKey', + path: [ + 'maxKey' + ], + count: 1, + type: 'MaxKey', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'MaxKey', + path: [ + 'maxKey' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + {} + ], + bsonType: 'MaxKey' + } + ] + }, + { + name: 'minKey', + path: [ + 'minKey' + ], + count: 1, + type: 'MinKey', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'MinKey', + path: [ + 'minKey' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + {} + ], + bsonType: 'MinKey' + } + ] + }, + { + name: 'null', + path: [ + 'null' + ], + count: 1, + type: 'Null', + probability: 0.8, + hasDuplicates: true, + types: [ + { + name: 'Null', + path: [ + 'null' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: true, + bsonType: 'Null' + } + ] + }, + { + name: 'object', + path: [ + 'object' + ], + count: 1, + type: 'Document', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Document', + path: [ + 'object' + ], + count: 1, + probability: 0.8, + bsonType: 'Document', + fields: [ + { + name: 'key', + path: [ + 'object', + 'key' + ], + count: 1, + type: 'String', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'String', + path: [ + 'object', + 'key' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'value' + ], + bsonType: 'String' + } + ] + } + ] + } + ] + }, + { + name: 'objectId', + path: [ + 'objectId' + ], + count: 1, + type: 'ObjectId', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'ObjectId', + path: [ + 'objectId' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + '642d766c7300158b1f22e975' + ], + bsonType: 'ObjectId' + } + ] + }, + { + name: 'regex', + path: [ + 'regex' + ], + count: 1, + type: 'BSONRegExp', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'BSONRegExp', + path: [ + 'regex' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + pattern: 'pattern', + options: 'i' + } + ], + bsonType: 'BSONRegExp' + } + ] + }, + { + name: 'string', + path: [ + 'string' + ], + count: 1, + type: 'String', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'String', + path: [ + 'string' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'Hello, world!' + ], + bsonType: 'String' + } + ] + }, + { + name: 'symbol', + path: [ + 'symbol' + ], + count: 1, + type: 'BSONSymbol', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'BSONSymbol', + path: [ + 'symbol' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'symbol' + ], + bsonType: 'BSONSymbol' + } + ] + }, + { + name: 'timestamp', + path: [ + 'timestamp' + ], + count: 1, + type: 'Timestamp', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Timestamp', + path: [ + 'timestamp' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + $timestamp: '7218556297505931265' + } + ], + bsonType: 'Timestamp' + } + ] + } + ] + }; + const standard = await internalSchemaToStandard(internal); + console.log(JSON.stringify(standard)); + assert.deepStrictEqual(standard, { + type: 'object', + required: [], + $defs: RELAXED_EJSON_DEFINITIONS, + properties: { + _id: { + $ref: '#/$defs/ObjectId' + }, + array: { + type: 'array', + items: { + $ref: '#/$defs/Double' + } + }, + binData: { + $ref: '#/$defs/Binary' + }, + binaries: { + type: 'object', + properties: { + binaryOld: { + $ref: '#/$defs/Binary' + }, + compressedTimeSeries: { + $ref: '#/$defs/Binary' + }, + custom: { + $ref: '#/$defs/Binary' + }, + encrypted: { + $ref: '#/$defs/Binary' + }, + functionData: { + $ref: '#/$defs/Binary' + }, + generic: { + $ref: '#/$defs/Binary' + }, + md5: { + $ref: '#/$defs/Binary' + }, + uuid: { + $ref: '#/$defs/Binary' + }, + uuidOld: { + $ref: '#/$defs/Binary' + } + }, + required: [] + }, + boolean: { + type: 'boolean' + }, + date: { + $ref: '#/$defs/Date' + }, + dbRef: { + $ref: '#/$defs/DBRef' + }, + decimal: { + $ref: '#/$defs/Decimal' + }, + double: { + $ref: '#/$defs/Double' + }, + int: { + type: 'integer' + }, + javascript: { + $ref: '#/$defs/Code' + }, + javascriptWithScope: { + $ref: '#/$defs/Code' + }, + long: { + type: 'integer' + }, + maxKey: { + $ref: '#/$defs/MaxKey' + }, + minKey: { + $ref: '#/$defs/MinKey' + }, + null: { + type: 'null' + }, + object: { + type: 'object', + properties: { + key: { + type: 'string' + } + } + }, + objectId: { + $ref: '#/$defs/ObjectId' + }, + regex: { + $ref: '#/$defs/RegExp' + }, + string: { + type: 'string' + }, + symbol: { + $ref: '#/$defs/BSONSymbol' + }, + timestamp: { + $ref: '#/$defs/Timestamp' + } + } + }); + }); + + it('nested document/object', async function() { + const internal = { + count: 2, + fields: [ + { + name: 'author', + path: [ + 'author' + ], + count: 1, + type: [ + 'Document', + 'Undefined' + ], + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'Document', + path: [ + 'author' + ], + count: 1, + probability: 0.5, + bsonType: 'Document', + fields: [ + { + name: 'name', + path: [ + 'author', + 'name' + ], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: [ + 'author', + 'name' + ], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: [ + 'Peter Sonder' + ], + bsonType: 'String' + } + ] + }, + { + name: 'rating', + path: [ + 'author', + 'rating' + ], + count: 1, + type: 'Double', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'Double', + path: [ + 'author', + 'rating' + ], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: [ + 1.3 + ], + bsonType: 'Double' + } + ] + } + ] + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: [ + 'author' + ], + count: 1, + probability: 0.5 + } + ] + } + ] + }; + const standard = await internalSchemaToStandard(internal); + assert.deepStrictEqual(standard, { + type: 'object', + required: ['author'], + $defs: RELAXED_EJSON_DEFINITIONS, + properties: { + author: { + type: 'object', + required: ['name', 'rating'], + properties: { + name: { + type: 'string' + }, + rating: { + $ref: '#/$defs/Double' + } + } + } + } + }); + }); + + describe('arrays', async function() { + it('array - single type', async function() { + const internal = { + count: 2, + fields: [ + { + name: 'genres', + path: [ + 'genres' + ], + count: 1, + type: [ + 'array', + 'Undefined' + ], + probability: 0.5, + hasDuplicates: false, + types: [ + { + name: 'array', + path: [ + 'genres' + ], + count: 1, + probability: 0.5, + bsonType: 'Array', + types: [ + { + name: 'String', + path: [ + 'genres' + ], + count: 2, + probability: 1, + unique: 2, + hasDuplicates: false, + values: [ + 'crimi', + 'comedy' + ], + bsonType: 'String' + } + ], + totalCount: 2, + lengths: [ + 2 + ], + averageLength: 2 + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: [ + 'genres' + ], + count: 1, + probability: 0.5 + } + ] + } + ] + }; + const standard = await internalSchemaToStandard(internal); + assert.deepStrictEqual(standard, { + type: 'object', + required: [], + $defs: RELAXED_EJSON_DEFINITIONS, + properties: { + genres: { + type: 'array', + items: { + type: 'string' + } + } + } + }); + }); + + it('array - complex mixed type', async function() { + const internal = { + count: 2, + fields: [ + { + name: 'genres', + path: [ + 'genres' + ], + count: 1, + type: [ + 'Array', + 'Undefined' + ], + probability: 0.5, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: [ + 'genres' + ], + count: 1, + probability: 0.5, + bsonType: 'Array', + types: [ + { + name: 'String', + path: [ + 'genres' + ], + count: 2, + probability: 0.6666666666666666, + unique: 2, + hasDuplicates: false, + values: [ + 'crimi', + 'comedy' + ], + bsonType: 'String' + }, + { + name: 'Document', + path: [ + 'genres' + ], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Document', + fields: [ + { + name: 'long', + path: [ + 'genres', + 'long' + ], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: [ + 'genres', + 'long' + ], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: [ + 'science fiction' + ], + bsonType: 'String' + } + ] + }, + { + name: 'short', + path: [ + 'genres', + 'short' + ], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: [ + 'genres', + 'short' + ], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: [ + 'scifi' + ], + bsonType: 'String' + } + ] + } + ] + } + ], + totalCount: 3, + lengths: [ + 3 + ], + averageLength: 3 + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: [ + 'genres' + ], + count: 1, + probability: 0.5 + } + ] + } + ] + }; + const standard = await internalSchemaToStandard(internal); + assert.deepStrictEqual(standard, { + type: 'object', + required: [], + $defs: RELAXED_EJSON_DEFINITIONS, + properties: { + genres: { + type: 'array', + items: { + anyOf: [ + { + type: 'string' + }, + { + type: 'object', + required: ['long', 'short'], + properties: { + long: { + type: 'string' + }, + short: { + type: 'string' + } + } + } + ] + } + } + } + }); + }); + + it('array - simple mixed type', async function() { + const internal = { + count: 2, + fields: [ + { + name: 'arrayMixedType', + path: [ + 'arrayMixedType' + ], + count: 1, + type: 'Array', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: [ + 'arrayMixedType' + ], + count: 1, + probability: 1, + bsonType: 'Array', + types: [ + { + name: 'int32', + path: [ + 'arrayMixedType' + ], + count: 2, + probability: 0.6666666666666666, + unique: 2, + hasDuplicates: false, + values: [ + 1, + 3 + ], + bsonType: 'Int32' + }, + { + name: 'String', + path: [ + 'arrayMixedType' + ], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: [ + '2' + ], + bsonType: 'String' + } + ], + totalCount: 3, + lengths: [ + 3 + ], + averageLength: 3 + } + ] + } + ] + }; + const standard = await internalSchemaToStandard(internal); + assert.deepStrictEqual(standard, { + type: 'object', + required: ['arrayMixedType'], + $defs: RELAXED_EJSON_DEFINITIONS, + properties: { + arrayMixedType: { + type: 'array', + items: { + type: ['integer', 'string'] + } + } + } + }); + }); + }); + + describe('mixed types', async function() { + it('simple mixed type', async function() { + const internal = { + count: 2, + fields: [ + { + name: 'mixedType', + path: [ + 'mixedType' + ], + count: 2, + type: [ + 'Int32', + 'String', + 'Undefined' + ], + probability: 0.6666666666666666, + hasDuplicates: false, + types: [ + { + name: 'Int32', + path: [ + 'mixedType' + ], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: [ + 1 + ], + bsonType: 'Int32' + }, + { + name: 'String', + path: [ + 'mixedType' + ], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: [ + 'abc' + ], + bsonType: 'String' + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: [ + 'mixedType' + ], + count: 1, + probability: 0.3333333333333333 + } + ] + } + ] + }; + const standard = await internalSchemaToStandard(internal); + assert.deepStrictEqual(standard, { + type: 'object', + required: [], + $defs: RELAXED_EJSON_DEFINITIONS, + properties: { + mixedType: { + type: ['integer', 'string'] + } + } + }); + }); + + it('complex mixed type', async function() { + const internal = { + count: 2, + fields: [ + { + name: 'mixedComplexType', + path: [ + 'mixedComplexType' + ], + count: 2, + type: [ + 'Array', + 'Document', + 'Undefined' + ], + probability: 0.6666666666666666, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: [ + 'mixedComplexType' + ], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Array', + types: [ + { + name: 'Int32', + path: [ + 'mixedComplexType' + ], + count: 3, + probability: 1, + unique: 3, + hasDuplicates: false, + values: [ + 1, + 2, + 3 + ], + bsonType: 'Int32' + } + ], + totalCount: 3, + lengths: [ + 3 + ], + averageLength: 3 + }, + { + name: 'Document', + path: [ + 'mixedComplexType' + ], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Document', + fields: [ + { + name: 'a', + path: [ + 'mixedComplexType', + 'a' + ], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: [ + 'mixedComplexType', + 'a' + ], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: [ + 'bc' + ], + bsonType: 'String' + } + ] + } + ] + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: [ + 'mixedComplexType' + ], + count: 1, + probability: 0.3333333333333333 + } + ] + } + ] + }; + const standard = await internalSchemaToStandard(internal); + assert.deepStrictEqual(standard, { + type: 'object', + required: [], + $defs: RELAXED_EJSON_DEFINITIONS, + properties: { + mixedComplexType: { + anyOf: [ + { + type: 'array', + items: { + type: 'integer' + } + }, + { + type: 'object', + required: ['a'], + properties: { + a: { + type: 'string' + } + } + } + ] + } + } + }); + }); + }); + + it('can be aborted', async function() { + const internal = { + count: 2, + fields: [ + { + name: 'mixedComplexType', + path: [ + 'mixedComplexType' + ], + count: 2, + type: [ + 'Array', + 'Document', + 'Undefined' + ], + probability: 0.6666666666666666, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: [ + 'mixedComplexType' + ], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Array', + types: [ + { + name: 'Int32', + path: [ + 'mixedComplexType' + ], + count: 3, + probability: 1, + unique: 3, + hasDuplicates: false, + values: [ + 1, + 2, + 3 + ], + bsonType: 'Int32' + } + ], + totalCount: 3, + lengths: [ + 3 + ], + averageLength: 3 + }, + { + name: 'Document', + path: [ + 'mixedComplexType' + ], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Document', + fields: [ + { + name: 'a', + path: [ + 'mixedComplexType', + 'a' + ], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: [ + 'mixedComplexType', + 'a' + ], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: [ + 'bc' + ], + bsonType: 'String' + } + ] + } + ] + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: [ + 'mixedComplexType' + ], + count: 1, + probability: 0.3333333333333333 + } + ] + } + ] + }; + const abortController = new AbortController(); + const promise = internalSchemaToStandard(internal, { signal: abortController.signal }); + abortController.abort(new Error('Too long, didn\'t wait.')); + await assert.rejects(promise, { + name: 'Error', + message: 'Too long, didn\'t wait.' + }); + }); + }); +}); diff --git a/src/schema-convertors/internalToStandard.ts b/src/schema-convertors/internalToStandard.ts index 631bab2..5ad1e72 100644 --- a/src/schema-convertors/internalToStandard.ts +++ b/src/schema-convertors/internalToStandard.ts @@ -1,21 +1,23 @@ +import { JSONSchema4TypeName } from 'json-schema'; import { ArraySchemaType, DocumentSchemaType, Schema as InternalSchema, SchemaType } from '../schema-analyzer'; import { StandardJSONSchema } from '../types'; +type StandardTypeDefinition = { type: JSONSchema4TypeName, $ref?: never; } | { $ref: string, type?: never }; + const InternalTypeToStandardTypeMap: Record< - SchemaType['name'] | 'Double' | 'BSONSymbol', - string | { $ref: string } + SchemaType['name'] | 'Double' | 'BSONSymbol', StandardTypeDefinition > = { Double: { $ref: '#/$defs/Double' }, Number: { $ref: '#/$defs/Double' }, - String: 'string', - Document: 'object', - Array: 'array', + String: { type: 'string' }, + Document: { type: 'object' }, + Array: { type: 'array' }, Binary: { $ref: '#/$defs/Binary' }, Undefined: { $ref: '#/$defs/Undefined' }, ObjectId: { $ref: '#/$defs/ObjectId' }, - Boolean: 'boolean', + Boolean: { type: 'boolean' }, Date: { $ref: '#/$defs/Date' }, - Null: 'null', + Null: { type: 'null' }, RegExp: { $ref: '#/$defs/RegExp' }, BSONRegExp: { $ref: '#/$defs/RegExp' }, DBRef: { $ref: '#/$defs/DBRef' }, @@ -23,15 +25,15 @@ const InternalTypeToStandardTypeMap: Record< BSONSymbol: { $ref: '#/$defs/BSONSymbol' }, Symbol: { $ref: '#/$defs/BSONSymbol' }, Code: { $ref: '#/$defs/Code' }, - Int32: 'integer', + Int32: { type: 'integer' }, Timestamp: { $ref: '#/$defs/Timestamp' }, - Long: 'integer', + Long: { type: 'integer' }, Decimal128: { $ref: '#/$defs/Decimal' }, MinKey: { $ref: '#/$defs/MinKey' }, MaxKey: { $ref: '#/$defs/MaxKey' } }; -const RELAXED_EJSON_DEFINITIONS = Object.freeze({ +export const RELAXED_EJSON_DEFINITIONS = Object.freeze({ ObjectId: { type: 'object', properties: { @@ -238,10 +240,10 @@ const RELAXED_EJSON_DEFINITIONS = Object.freeze({ } }); -const convertInternalType = (type: string) => { - const bsonType = InternalTypeToStandardTypeMap[type]; - if (!bsonType) throw new Error(`Encountered unknown type: ${type}`); - return bsonType; +const convertInternalType = (internalType: string) => { + const type = InternalTypeToStandardTypeMap[internalType]; + if (!type) throw new Error(`Encountered unknown type: ${internalType}`); + return type; }; async function allowAbort(signal?: AbortSignal) { @@ -255,9 +257,7 @@ async function allowAbort(signal?: AbortSignal) { async function parseType(type: SchemaType, signal?: AbortSignal): Promise { await allowAbort(signal); - const schema: StandardJSONSchema = { - bsonType: convertInternalType(type.bsonType) - }; + const schema: StandardJSONSchema = convertInternalType(type.bsonType); switch (type.bsonType) { case 'Array': schema.items = await parseTypes((type as ArraySchemaType).types); @@ -272,6 +272,10 @@ async function parseType(type: SchemaType, signal?: AbortSignal): Promise !definition.$ref); +} + async function parseTypes(types: SchemaType[], signal?: AbortSignal): Promise { await allowAbort(signal); const definedTypes = types.filter(type => type.bsonType.toLowerCase() !== 'undefined'); @@ -285,8 +289,14 @@ async function parseTypes(types: SchemaType[], signal?: AbortSignal): Promise convertInternalType(type.bsonType)); + if (isSimpleTypesOnly(convertedTypes)) { + return { + type: convertedTypes.map(({ type }) => type) + }; + } return { - bsonType: definedTypes.map((type) => convertInternalType(type.bsonType)) + anyOf: convertedTypes }; } @@ -311,7 +321,7 @@ export default async function internalSchemaToMongodb( } = {}): Promise { const { required, properties } = await parseFields(internalSchema.fields, options.signal); const schema: StandardJSONSchema = { - bsonType: 'object', + type: 'object', required, properties, $defs: RELAXED_EJSON_DEFINITIONS From 546941c4fe3198e7cbb4af5a626349487e356d78 Mon Sep 17 00:00:00 2001 From: Paula Stachova Date: Mon, 3 Feb 2025 16:33:38 +0100 Subject: [PATCH 03/13] add tests and fix --- src/schema-convertors/internalToMongoDB.ts | 10 ++- .../internalToStandard.test.ts | 74 ++++++++++++++++++- src/schema-convertors/internalToStandard.ts | 18 ++--- 3 files changed, 83 insertions(+), 19 deletions(-) diff --git a/src/schema-convertors/internalToMongoDB.ts b/src/schema-convertors/internalToMongoDB.ts index 7727c84..434daa0 100644 --- a/src/schema-convertors/internalToMongoDB.ts +++ b/src/schema-convertors/internalToMongoDB.ts @@ -64,6 +64,10 @@ async function parseType(type: SchemaType, signal?: AbortSignal): Promise !!definition.bsonType && Object.keys(definition).length === 1); +} + async function parseTypes(types: SchemaType[], signal?: AbortSignal): Promise { await allowAbort(signal); const definedTypes = types.filter(type => type.bsonType.toLowerCase() !== 'undefined'); @@ -72,13 +76,13 @@ async function parseTypes(types: SchemaType[], signal?: AbortSignal): Promise parseType(type, signal))); - if (definedTypes.some(type => ['Document', 'Array'].includes(type.bsonType))) { + if (isSimpleTypesOnly(parsedTypes)) { return { - anyOf: parsedTypes + bsonType: parsedTypes.map(({ bsonType }) => bsonType) }; } return { - bsonType: definedTypes.map((type) => convertInternalType(type.bsonType)) + anyOf: parsedTypes }; } diff --git a/src/schema-convertors/internalToStandard.test.ts b/src/schema-convertors/internalToStandard.test.ts index 99acba4..ca37874 100644 --- a/src/schema-convertors/internalToStandard.test.ts +++ b/src/schema-convertors/internalToStandard.test.ts @@ -3,7 +3,7 @@ import internalSchemaToStandard, { RELAXED_EJSON_DEFINITIONS } from './internalT describe('internalSchemaToStandard', async function() { describe('Converts: ', async function() { - it.only('all the types', async function() { + it('all the types', async function() { const internal = { count: 1, fields: [ @@ -893,7 +893,6 @@ describe('internalSchemaToStandard', async function() { ] }; const standard = await internalSchemaToStandard(internal); - console.log(JSON.stringify(standard)); assert.deepStrictEqual(standard, { type: 'object', required: [], @@ -986,7 +985,8 @@ describe('internalSchemaToStandard', async function() { key: { type: 'string' } - } + }, + required: [] }, objectId: { $ref: '#/$defs/ObjectId' @@ -1525,7 +1525,7 @@ describe('internalSchemaToStandard', async function() { }); }); - it('complex mixed type', async function() { + it('complex mixed type (with array and object)', async function() { const internal = { count: 2, fields: [ @@ -1657,6 +1657,72 @@ describe('internalSchemaToStandard', async function() { } }); }); + + it('complex mixed type (with $refs)', async function() { + const internal = { + count: 2, + fields: [ + { + name: 'mixedType', + path: [ + 'mixedType' + ], + count: 2, + type: [ + 'String', + 'ObjectId' + ], + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: [ + 'mixedType' + ], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: [ + 'abc' + ], + bsonType: 'String' + }, + { + name: 'ObjectId', + path: [ + 'objectId' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + '642d766c7300158b1f22e975' + ], + bsonType: 'ObjectId' + } + ] + } + ] + }; + const standard = await internalSchemaToStandard(internal); + assert.deepStrictEqual(standard, { + type: 'object', + required: ['mixedType'], + $defs: RELAXED_EJSON_DEFINITIONS, + properties: { + mixedType: { + anyOf: [{ + type: 'string' + }, { + $ref: '#/$defs/ObjectId' + }] + } + } + }); + }); }); it('can be aborted', async function() { diff --git a/src/schema-convertors/internalToStandard.ts b/src/schema-convertors/internalToStandard.ts index 5ad1e72..04270d3 100644 --- a/src/schema-convertors/internalToStandard.ts +++ b/src/schema-convertors/internalToStandard.ts @@ -241,7 +241,7 @@ export const RELAXED_EJSON_DEFINITIONS = Object.freeze({ }); const convertInternalType = (internalType: string) => { - const type = InternalTypeToStandardTypeMap[internalType]; + const type = { ...InternalTypeToStandardTypeMap[internalType] }; if (!type) throw new Error(`Encountered unknown type: ${internalType}`); return type; }; @@ -272,8 +272,8 @@ async function parseType(type: SchemaType, signal?: AbortSignal): Promise !definition.$ref); +function isSimpleTypesOnly(types: StandardJSONSchema[]): types is { type: JSONSchema4TypeName }[] { + return types.every(definition => !!definition.type && Object.keys(definition).length === 1); } async function parseTypes(types: SchemaType[], signal?: AbortSignal): Promise { @@ -284,19 +284,13 @@ async function parseTypes(types: SchemaType[], signal?: AbortSignal): Promise parseType(type, signal))); - if (definedTypes.some(type => ['Document', 'Array'].includes(type.bsonType))) { + if (isSimpleTypesOnly(parsedTypes)) { return { - anyOf: parsedTypes - }; - } - const convertedTypes = definedTypes.map((type) => convertInternalType(type.bsonType)); - if (isSimpleTypesOnly(convertedTypes)) { - return { - type: convertedTypes.map(({ type }) => type) + type: parsedTypes.map(({ type }) => type) }; } return { - anyOf: convertedTypes + anyOf: parsedTypes }; } From 3e6046da757391f0dc12faaec4b5822035fafd9f Mon Sep 17 00:00:00 2001 From: Paula Stachova Date: Mon, 3 Feb 2025 16:35:18 +0100 Subject: [PATCH 04/13] rename --- src/schema-convertors/internalToMongoDB.ts | 4 ++-- src/schema-convertors/internalToStandard.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/schema-convertors/internalToMongoDB.ts b/src/schema-convertors/internalToMongoDB.ts index 434daa0..0aaf952 100644 --- a/src/schema-convertors/internalToMongoDB.ts +++ b/src/schema-convertors/internalToMongoDB.ts @@ -64,7 +64,7 @@ async function parseType(type: SchemaType, signal?: AbortSignal): Promise !!definition.bsonType && Object.keys(definition).length === 1); } @@ -76,7 +76,7 @@ async function parseTypes(types: SchemaType[], signal?: AbortSignal): Promise parseType(type, signal))); - if (isSimpleTypesOnly(parsedTypes)) { + if (isPlainTypesOnly(parsedTypes)) { return { bsonType: parsedTypes.map(({ bsonType }) => bsonType) }; diff --git a/src/schema-convertors/internalToStandard.ts b/src/schema-convertors/internalToStandard.ts index 04270d3..8e4bc51 100644 --- a/src/schema-convertors/internalToStandard.ts +++ b/src/schema-convertors/internalToStandard.ts @@ -272,7 +272,7 @@ async function parseType(type: SchemaType, signal?: AbortSignal): Promise !!definition.type && Object.keys(definition).length === 1); } @@ -284,7 +284,7 @@ async function parseTypes(types: SchemaType[], signal?: AbortSignal): Promise parseType(type, signal))); - if (isSimpleTypesOnly(parsedTypes)) { + if (isPlainTypesOnly(parsedTypes)) { return { type: parsedTypes.map(({ type }) => type) }; From f6f21408aba543e5194dd158324b5542d591a2f6 Mon Sep 17 00:00:00 2001 From: Paula Stachova Date: Mon, 3 Feb 2025 16:36:51 +0100 Subject: [PATCH 05/13] cleanup --- src/schema-convertors/internalToStandard.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/schema-convertors/internalToStandard.ts b/src/schema-convertors/internalToStandard.ts index 8e4bc51..0086000 100644 --- a/src/schema-convertors/internalToStandard.ts +++ b/src/schema-convertors/internalToStandard.ts @@ -111,7 +111,7 @@ export const RELAXED_EJSON_DEFINITIONS = Object.freeze({ type: 'string' }, $scope: { - type: 'object' // TODO: object is ejson object hmm + type: 'object' } }, required: ['$code', '$scope'], From fd973d5a2b0c4166bef2e3cbe1dddbcba9a4b419 Mon Sep 17 00:00:00 2001 From: Paula Stachova Date: Mon, 3 Feb 2025 17:53:55 +0100 Subject: [PATCH 06/13] rename --- src/schema-convertors/internalToStandard.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/schema-convertors/internalToStandard.ts b/src/schema-convertors/internalToStandard.ts index 0086000..0ec1671 100644 --- a/src/schema-convertors/internalToStandard.ts +++ b/src/schema-convertors/internalToStandard.ts @@ -308,7 +308,7 @@ async function parseFields(fields: DocumentSchemaType['fields'], signal?: AbortS return { required, properties }; } -export default async function internalSchemaToMongodb( +export default async function internalSchemaToStandard( internalSchema: InternalSchema, options: { signal?: AbortSignal From 0596dc8ec8e6624efbd0a2a0dc38569f33deac7b Mon Sep 17 00:00:00 2001 From: Paula Stachova Date: Tue, 4 Feb 2025 14:31:44 +0100 Subject: [PATCH 07/13] add schema validation against meta --- package-lock.json | 240 +++++++++++++++--- package.json | 1 + .../internalToStandard.test.ts | 19 ++ src/schema-convertors/internalToStandard.ts | 3 +- 4 files changed, 228 insertions(+), 35 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5f9e9cd..afb667b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "@types/sinon": "^17.0.3", "@typescript-eslint/eslint-plugin": "^5.47.1", "@typescript-eslint/parser": "^5.47.1", + "ajv": "^8.17.1", "bson": "^6.7.0", "coveralls": "^3.1.1", "depcheck": "^1.4.3", @@ -1189,6 +1190,23 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@eslint/eslintrc/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/@eslint/eslintrc/node_modules/globals": { "version": "13.19.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.19.0.tgz", @@ -1204,6 +1222,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, "node_modules/@eslint/eslintrc/node_modules/type-fest": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", @@ -2805,15 +2830,16 @@ } }, "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dev": true, + "license": "MIT", "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" }, "funding": { "type": "github", @@ -4027,6 +4053,23 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/eslint/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/eslint/node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -4131,6 +4174,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/eslint/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, "node_modules/eslint/node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -4316,10 +4366,11 @@ } }, "node_modules/fast-json-stable-stringify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=", - "dev": true + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" }, "node_modules/fast-levenshtein": { "version": "2.0.6", @@ -4327,6 +4378,23 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "node_modules/fast-uri": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.6.tgz", + "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fast-xml-parser": { "version": "4.2.5", "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz", @@ -4737,6 +4805,30 @@ "node": ">=6" } }, + "node_modules/har-validator/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/har-validator/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, "node_modules/has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", @@ -5449,10 +5541,11 @@ "dev": true }, "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -6812,6 +6905,16 @@ "node": ">=0.10.0" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/require-main-filename": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", @@ -7561,10 +7664,11 @@ "license": "MIT" }, "node_modules/uri-js": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", "dev": true, + "license": "BSD-2-Clause", "dependencies": { "punycode": "^2.1.0" } @@ -8968,6 +9072,18 @@ "strip-json-comments": "^3.1.1" }, "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, "globals": { "version": "13.19.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.19.0.tgz", @@ -8977,6 +9093,12 @@ "type-fest": "^0.20.2" } }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "type-fest": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", @@ -10381,15 +10503,15 @@ } }, "ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dev": true, "requires": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" } }, "ansi-colors": { @@ -11109,6 +11231,18 @@ "text-table": "^0.2.0" }, "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, "ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -11177,6 +11311,12 @@ "type-fest": "^0.20.2" } }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -11518,9 +11658,9 @@ } }, "fast-json-stable-stringify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", "dev": true }, "fast-levenshtein": { @@ -11529,6 +11669,12 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "fast-uri": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.6.tgz", + "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==", + "dev": true + }, "fast-xml-parser": { "version": "4.2.5", "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz", @@ -11815,6 +11961,26 @@ "requires": { "ajv": "^6.12.3", "har-schema": "^2.0.0" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + } } }, "has": { @@ -12345,9 +12511,9 @@ "dev": true }, "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "dev": true }, "json-stable-stringify-without-jsonify": { @@ -13337,6 +13503,12 @@ "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", "devOptional": true }, + "require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true + }, "require-main-filename": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", @@ -13894,9 +14066,9 @@ "dev": true }, "uri-js": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", "dev": true, "requires": { "punycode": "^2.1.0" diff --git a/package.json b/package.json index 5bfc874..8d0dac6 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,7 @@ "@types/sinon": "^17.0.3", "@typescript-eslint/eslint-plugin": "^5.47.1", "@typescript-eslint/parser": "^5.47.1", + "ajv": "^8.17.1", "bson": "^6.7.0", "coveralls": "^3.1.1", "depcheck": "^1.4.3", diff --git a/src/schema-convertors/internalToStandard.test.ts b/src/schema-convertors/internalToStandard.test.ts index ca37874..4f08100 100644 --- a/src/schema-convertors/internalToStandard.test.ts +++ b/src/schema-convertors/internalToStandard.test.ts @@ -1,7 +1,10 @@ import assert from 'assert'; +import Ajv2020 from 'ajv'; import internalSchemaToStandard, { RELAXED_EJSON_DEFINITIONS } from './internalToStandard'; describe('internalSchemaToStandard', async function() { + const ajv = new Ajv2020(); + describe('Converts: ', async function() { it('all the types', async function() { const internal = { @@ -893,7 +896,9 @@ describe('internalSchemaToStandard', async function() { ] }; const standard = await internalSchemaToStandard(internal); + ajv.validateSchema(standard); assert.deepStrictEqual(standard, { + $schema: 'https://json-schema.org/draft/2020-12/schema', type: 'object', required: [], $defs: RELAXED_EJSON_DEFINITIONS, @@ -1107,7 +1112,9 @@ describe('internalSchemaToStandard', async function() { ] }; const standard = await internalSchemaToStandard(internal); + ajv.validateSchema(standard); assert.deepStrictEqual(standard, { + $schema: 'https://json-schema.org/draft/2020-12/schema', type: 'object', required: ['author'], $defs: RELAXED_EJSON_DEFINITIONS, @@ -1193,7 +1200,9 @@ describe('internalSchemaToStandard', async function() { ] }; const standard = await internalSchemaToStandard(internal); + ajv.validateSchema(standard); assert.deepStrictEqual(standard, { + $schema: 'https://json-schema.org/draft/2020-12/schema', type: 'object', required: [], $defs: RELAXED_EJSON_DEFINITIONS, @@ -1339,7 +1348,9 @@ describe('internalSchemaToStandard', async function() { ] }; const standard = await internalSchemaToStandard(internal); + ajv.validateSchema(standard); assert.deepStrictEqual(standard, { + $schema: 'https://json-schema.org/draft/2020-12/schema', type: 'object', required: [], $defs: RELAXED_EJSON_DEFINITIONS, @@ -1434,7 +1445,9 @@ describe('internalSchemaToStandard', async function() { ] }; const standard = await internalSchemaToStandard(internal); + ajv.validateSchema(standard); assert.deepStrictEqual(standard, { + $schema: 'https://json-schema.org/draft/2020-12/schema', type: 'object', required: ['arrayMixedType'], $defs: RELAXED_EJSON_DEFINITIONS, @@ -1513,7 +1526,9 @@ describe('internalSchemaToStandard', async function() { ] }; const standard = await internalSchemaToStandard(internal); + ajv.validateSchema(standard); assert.deepStrictEqual(standard, { + $schema: 'https://json-schema.org/draft/2020-12/schema', type: 'object', required: [], $defs: RELAXED_EJSON_DEFINITIONS, @@ -1630,7 +1645,9 @@ describe('internalSchemaToStandard', async function() { ] }; const standard = await internalSchemaToStandard(internal); + ajv.validateSchema(standard); assert.deepStrictEqual(standard, { + $schema: 'https://json-schema.org/draft/2020-12/schema', type: 'object', required: [], $defs: RELAXED_EJSON_DEFINITIONS, @@ -1708,7 +1725,9 @@ describe('internalSchemaToStandard', async function() { ] }; const standard = await internalSchemaToStandard(internal); + ajv.validateSchema(standard); assert.deepStrictEqual(standard, { + $schema: 'https://json-schema.org/draft/2020-12/schema', type: 'object', required: ['mixedType'], $defs: RELAXED_EJSON_DEFINITIONS, diff --git a/src/schema-convertors/internalToStandard.ts b/src/schema-convertors/internalToStandard.ts index 0ec1671..049a7d2 100644 --- a/src/schema-convertors/internalToStandard.ts +++ b/src/schema-convertors/internalToStandard.ts @@ -4,7 +4,7 @@ import { StandardJSONSchema } from '../types'; type StandardTypeDefinition = { type: JSONSchema4TypeName, $ref?: never; } | { $ref: string, type?: never }; -const InternalTypeToStandardTypeMap: Record< +export const InternalTypeToStandardTypeMap: Record< SchemaType['name'] | 'Double' | 'BSONSymbol', StandardTypeDefinition > = { Double: { $ref: '#/$defs/Double' }, @@ -315,6 +315,7 @@ export default async function internalSchemaToStandard( } = {}): Promise { const { required, properties } = await parseFields(internalSchema.fields, options.signal); const schema: StandardJSONSchema = { + $schema: 'https://json-schema.org/draft/2020-12/schema', type: 'object', required, properties, From e7e599b800016f6ae4dc7f196389e70b4398d1b9 Mon Sep 17 00:00:00 2001 From: Paula Stachova Date: Tue, 4 Feb 2025 15:14:11 +0100 Subject: [PATCH 08/13] adress pr comments --- src/schema-accessor.ts | 10 ++++++++++ src/schema-convertors/internalToMongoDB.ts | 15 +++++---------- src/schema-convertors/internalToStandard.test.ts | 2 +- src/schema-convertors/internalToStandard.ts | 10 +--------- src/schema-convertors/util.ts | 8 ++++++++ 5 files changed, 25 insertions(+), 20 deletions(-) create mode 100644 src/schema-convertors/util.ts diff --git a/src/schema-accessor.ts b/src/schema-accessor.ts index 88f79e6..edf6128 100644 --- a/src/schema-accessor.ts +++ b/src/schema-accessor.ts @@ -33,14 +33,24 @@ export class InternalSchemaBasedAccessor implements SchemaAccessor { return this.internalSchema; } + /** + * Get standard JSON Schema - as per + * https://json-schema.org/draft/2020-12/schema + */ async getStandardJsonSchema(options: Options = {}): Promise { return this.standardJSONSchema ??= await convertors.internalSchemaToStandard(this.internalSchema, options); } + /** + * Get MongoDB's $jsonSchema + */ async getMongoDBJsonSchema(options: Options = {}): Promise { return this.mongodbJSONSchema ??= await convertors.internalSchemaToMongoDB(this.internalSchema, options); } + /** + * Get expanded JSON Schema - with additional properties + */ async getExpandedJSONSchema(options: Options = {}): Promise { return this.ExpandedJSONSchema ??= await convertors.internalSchemaToExpanded(this.internalSchema, options); } diff --git a/src/schema-convertors/internalToMongoDB.ts b/src/schema-convertors/internalToMongoDB.ts index 0aaf952..5d6875b 100644 --- a/src/schema-convertors/internalToMongoDB.ts +++ b/src/schema-convertors/internalToMongoDB.ts @@ -1,7 +1,11 @@ +/** + * Transforms the internal schema to $jsonSchema + */ import { ArraySchemaType, DocumentSchemaType, Schema as InternalSchema, SchemaType } from '../schema-analyzer'; import { MongoDBJSONSchema } from '../types'; +import { allowAbort } from './util'; -const InternalTypeToBsonTypeMap: Record< +export const InternalTypeToBsonTypeMap: Record< SchemaType['name'] | 'Double' | 'BSONSymbol', string > = { @@ -36,15 +40,6 @@ const convertInternalType = (type: string) => { return bsonType; }; -async function allowAbort(signal?: AbortSignal) { - return new Promise((resolve, reject) => - setTimeout(() => { - if (signal?.aborted) return reject(signal?.reason || new Error('Operation aborted')); - resolve(); - }) - ); -} - async function parseType(type: SchemaType, signal?: AbortSignal): Promise { await allowAbort(signal); const schema: MongoDBJSONSchema = { diff --git a/src/schema-convertors/internalToStandard.test.ts b/src/schema-convertors/internalToStandard.test.ts index 4f08100..614c8bc 100644 --- a/src/schema-convertors/internalToStandard.test.ts +++ b/src/schema-convertors/internalToStandard.test.ts @@ -1,5 +1,5 @@ import assert from 'assert'; -import Ajv2020 from 'ajv'; +import Ajv2020 from 'ajv/dist/2020'; import internalSchemaToStandard, { RELAXED_EJSON_DEFINITIONS } from './internalToStandard'; describe('internalSchemaToStandard', async function() { diff --git a/src/schema-convertors/internalToStandard.ts b/src/schema-convertors/internalToStandard.ts index 049a7d2..ad86442 100644 --- a/src/schema-convertors/internalToStandard.ts +++ b/src/schema-convertors/internalToStandard.ts @@ -1,6 +1,7 @@ import { JSONSchema4TypeName } from 'json-schema'; import { ArraySchemaType, DocumentSchemaType, Schema as InternalSchema, SchemaType } from '../schema-analyzer'; import { StandardJSONSchema } from '../types'; +import { allowAbort } from './util'; type StandardTypeDefinition = { type: JSONSchema4TypeName, $ref?: never; } | { $ref: string, type?: never }; @@ -246,15 +247,6 @@ const convertInternalType = (internalType: string) => { return type; }; -async function allowAbort(signal?: AbortSignal) { - return new Promise((resolve, reject) => - setTimeout(() => { - if (signal?.aborted) return reject(signal?.reason || new Error('Operation aborted')); - resolve(); - }) - ); -} - async function parseType(type: SchemaType, signal?: AbortSignal): Promise { await allowAbort(signal); const schema: StandardJSONSchema = convertInternalType(type.bsonType); diff --git a/src/schema-convertors/util.ts b/src/schema-convertors/util.ts new file mode 100644 index 0000000..941ceee --- /dev/null +++ b/src/schema-convertors/util.ts @@ -0,0 +1,8 @@ +export async function allowAbort(signal?: AbortSignal) { + return new Promise((resolve, reject) => + setTimeout(() => { + if (signal?.aborted) return reject(signal?.reason || new Error('Operation aborted')); + resolve(); + }) + ); +} From c3f38eb28e13d0ec5d4de69e669fe9ea725971af Mon Sep 17 00:00:00 2001 From: Paula Stachova Date: Wed, 5 Feb 2025 10:52:28 +0100 Subject: [PATCH 09/13] fix double definition --- src/schema-convertors/internalToStandard.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/schema-convertors/internalToStandard.ts b/src/schema-convertors/internalToStandard.ts index ad86442..975c052 100644 --- a/src/schema-convertors/internalToStandard.ts +++ b/src/schema-convertors/internalToStandard.ts @@ -60,7 +60,12 @@ export const RELAXED_EJSON_DEFINITIONS = Object.freeze({ oneOf: [ { type: 'number' }, { - enum: ['Infinity', '-Infinity', 'NaN'] + type: 'object', + properties: { + $numberDouble: { + enum: ['Infinity', '-Infinity', 'NaN'] + } + } } ] }, From d922418ef2fd7e5c9e6f71cb857f810d4d5705ce Mon Sep 17 00:00:00 2001 From: Paula Stachova Date: Wed, 5 Feb 2025 13:21:05 +0100 Subject: [PATCH 10/13] address PR comments --- src/schema-convertors/internalToStandard.test.ts | 2 +- src/schema-convertors/internalToStandard.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/schema-convertors/internalToStandard.test.ts b/src/schema-convertors/internalToStandard.test.ts index 614c8bc..ba00bb1 100644 --- a/src/schema-convertors/internalToStandard.test.ts +++ b/src/schema-convertors/internalToStandard.test.ts @@ -958,7 +958,7 @@ describe('internalSchemaToStandard', async function() { $ref: '#/$defs/DBRef' }, decimal: { - $ref: '#/$defs/Decimal' + $ref: '#/$defs/Decimal128' }, double: { $ref: '#/$defs/Double' diff --git a/src/schema-convertors/internalToStandard.ts b/src/schema-convertors/internalToStandard.ts index 975c052..5b5e4ab 100644 --- a/src/schema-convertors/internalToStandard.ts +++ b/src/schema-convertors/internalToStandard.ts @@ -24,12 +24,12 @@ export const InternalTypeToStandardTypeMap: Record< DBRef: { $ref: '#/$defs/DBRef' }, DBPointer: { $ref: '#/$defs/DBPointer' }, BSONSymbol: { $ref: '#/$defs/BSONSymbol' }, - Symbol: { $ref: '#/$defs/BSONSymbol' }, Code: { $ref: '#/$defs/Code' }, + CodeWScope: { $ref: '#/$defs/Code' }, Int32: { type: 'integer' }, Timestamp: { $ref: '#/$defs/Timestamp' }, Long: { type: 'integer' }, - Decimal128: { $ref: '#/$defs/Decimal' }, + Decimal128: { $ref: '#/$defs/Decimal128' }, MinKey: { $ref: '#/$defs/MinKey' }, MaxKey: { $ref: '#/$defs/MaxKey' } }; @@ -176,7 +176,7 @@ export const RELAXED_EJSON_DEFINITIONS = Object.freeze({ type: 'string' }, $id: { - $ref: '#/$defs/Decimal' + $ref: '#/$defs/ObjectId' } }, required: ['$ref', '$id'], From 37ec24c7a94dfb4dc5ea3eabf3eb5ee7f1486d3b Mon Sep 17 00:00:00 2001 From: Paula Stachova Date: Wed, 5 Feb 2025 13:40:55 +0100 Subject: [PATCH 11/13] add integration test --- package.json | 2 +- test/integration/generateAndValidate.ts | 45 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 test/integration/generateAndValidate.ts diff --git a/package.json b/package.json index 8d0dac6..47e3fdc 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ ".esm-wrapper.mjs" ], "scripts": { - "test": "nyc mocha --timeout 5000 --colors -r ts-node/register test/*.ts src/**/*.test.ts", + "test": "nyc mocha --timeout 5000 --colors -r ts-node/register test/**/*.ts src/**/*.test.ts", "test-example-parse-from-file": "ts-node examples/parse-from-file.ts", "test-example-parse-schema": "ts-node examples/parse-schema.ts", "test-time": "ts-node ./test/time-testing.ts", diff --git a/test/integration/generateAndValidate.ts b/test/integration/generateAndValidate.ts new file mode 100644 index 0000000..ce1623f --- /dev/null +++ b/test/integration/generateAndValidate.ts @@ -0,0 +1,45 @@ +import { analyzeDocuments } from '../../src'; +import Ajv2020 from 'ajv/dist/2020'; +import assert from 'assert'; + +const documents = [{ + _id: { + $oid: '67863e82fb817085a6b0ebad' + }, + title: 'My book', + year: 1983, + genres: [ + 'crimi', + 'comedy', + { + short: 'scifi', + long: 'science fiction' + } + ], + number: { + $numberDouble: 'Infinity' + } +}, +{ + _id: { + $oid: '67863eacfb817085a6b0ebae' + }, + title: 'Other book', + year: 1999, + author: { + name: 'Peter Sonder', + rating: 1.3 + } +}]; + +describe('Documents -> Generate schema -> Validate Documents against the schema', function() { + it('Standard JSON Schema with Relaxed EJSON', async function() { + const ajv = new Ajv2020(); + const analyzedDocuments = await analyzeDocuments(documents); + const schema = await analyzedDocuments.getStandardJsonSchema(); + const validate = ajv.compile(schema); + for (const doc of documents) { + assert.strictEqual(validate(doc), true); + } + }); +}); From 5f4b2686890eee0080ff4c90a64df357c0ae5915 Mon Sep 17 00:00:00 2001 From: Paula Stachova Date: Wed, 5 Feb 2025 14:12:27 +0100 Subject: [PATCH 12/13] better integration test --- test/integration/generateAndValidate.ts | 35 +++++++++++++------------ 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/test/integration/generateAndValidate.ts b/test/integration/generateAndValidate.ts index ce1623f..552a08f 100644 --- a/test/integration/generateAndValidate.ts +++ b/test/integration/generateAndValidate.ts @@ -1,13 +1,12 @@ import { analyzeDocuments } from '../../src'; import Ajv2020 from 'ajv/dist/2020'; import assert from 'assert'; +import { ObjectId, Int32, Double, EJSON } from 'bson'; -const documents = [{ - _id: { - $oid: '67863e82fb817085a6b0ebad' - }, +const bsonDocuments = [{ + _id: new ObjectId('67863e82fb817085a6b0ebad'), title: 'My book', - year: 1983, + year: new Int32(1983), genres: [ 'crimi', 'comedy', @@ -16,30 +15,32 @@ const documents = [{ long: 'science fiction' } ], - number: { - $numberDouble: 'Infinity' - } + number: Double.fromString('Infinity') }, { - _id: { - $oid: '67863eacfb817085a6b0ebae' - }, + _id: new ObjectId('67863eacfb817085a6b0ebae'), title: 'Other book', - year: 1999, + year: new Int32('1999'), author: { name: 'Peter Sonder', - rating: 1.3 + rating: new Double(1.3) } }]; -describe('Documents -> Generate schema -> Validate Documents against the schema', function() { +describe.only('Documents -> Generate schema -> Validate Documents against the schema', function() { it('Standard JSON Schema with Relaxed EJSON', async function() { const ajv = new Ajv2020(); - const analyzedDocuments = await analyzeDocuments(documents); + // First we get the schema + const analyzedDocuments = await analyzeDocuments(bsonDocuments); const schema = await analyzedDocuments.getStandardJsonSchema(); const validate = ajv.compile(schema); - for (const doc of documents) { - assert.strictEqual(validate(doc), true); + for (const doc of bsonDocuments) { + // Then we get EJSON documents + const relaxedEJSONDoc = EJSON.serialize(doc, { relaxed: true }); + // Which we validate against the schema + const valid = validate(relaxedEJSONDoc); + if (validate.errors) console.error('Validation failed', validate.errors); + assert.strictEqual(valid, true); } }); }); From b260d86f97ed809a8329e383992766bdff60b49f Mon Sep 17 00:00:00 2001 From: Paula Stachova Date: Wed, 5 Feb 2025 14:23:31 +0100 Subject: [PATCH 13/13] comments --- test/integration/generateAndValidate.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/generateAndValidate.ts b/test/integration/generateAndValidate.ts index 552a08f..de2c3c4 100644 --- a/test/integration/generateAndValidate.ts +++ b/test/integration/generateAndValidate.ts @@ -30,14 +30,14 @@ const bsonDocuments = [{ describe.only('Documents -> Generate schema -> Validate Documents against the schema', function() { it('Standard JSON Schema with Relaxed EJSON', async function() { const ajv = new Ajv2020(); - // First we get the schema + // First we get the JSON schema from BSON const analyzedDocuments = await analyzeDocuments(bsonDocuments); const schema = await analyzedDocuments.getStandardJsonSchema(); const validate = ajv.compile(schema); for (const doc of bsonDocuments) { - // Then we get EJSON documents + // Then we get EJSON const relaxedEJSONDoc = EJSON.serialize(doc, { relaxed: true }); - // Which we validate against the schema + // And validate it agains the JSON Schema const valid = validate(relaxedEJSONDoc); if (validate.errors) console.error('Validation failed', validate.errors); assert.strictEqual(valid, true);