Restructure TEST_TABLE for tests.

Author: rkistner

libs/lib-services/src/logger/logger-index.ts

@@ -1,2 +1,3 @@
 export * from './Logger.js';
 export { Logger } from 'winston';
+export { createLogger, format, transports } from 'winston';

libs/lib-services/src/migrations/AbstractMigrationAgent.ts

@@ -1,11 +1,12 @@
 import { LockManager } from '../locks/LockManager.js';
-import { logger } from '../logger/Logger.js';
+import { logger as defaultLogger, Logger } from '../logger/logger-index.js';
 import * as defs from './migration-definitions.js';
 
 export type MigrationParams<Generics extends MigrationAgentGenerics = MigrationAgentGenerics> = {
   count?: number;
   direction: defs.Direction;
   migrationContext?: Generics['MIGRATION_CONTEXT'];
+  logger?: Logger;
 };
 
 type WriteLogsParams = {
@@ -20,10 +21,12 @@ export type MigrationAgentGenerics = {
 export type RunMigrationParams<Generics extends MigrationAgentGenerics = MigrationAgentGenerics> = MigrationParams & {
   migrations: defs.Migration<Generics['MIGRATION_CONTEXT']>[];
   maxLockWaitMs?: number;
+  logger?: Logger;
 };
 
 type ExecuteParams = RunMigrationParams & {
   state?: defs.MigrationState;
+  logger: Logger;
 };
 
 export const DEFAULT_MAX_LOCK_WAIT_MS = 3 * 60 * 1000; // 3 minutes
@@ -46,9 +49,11 @@ export abstract class AbstractMigrationAgent<Generics extends MigrationAgentGene
   async run(params: RunMigrationParams) {
     await this.init();
 
+    const logger = params.logger ?? defaultLogger;
+
     const { direction, migrations, migrationContext } = params;
     // Only one process should execute this at a time.
-    logger.info('Acquiring lock for migrations');
+    logger.debug('Acquiring lock for migrations');
     const lockHandle = await this.locks.acquire({ max_wait_ms: params.maxLockWaitMs ?? DEFAULT_MAX_LOCK_WAIT_MS });
 
     if (!lockHandle) {
@@ -75,22 +80,24 @@ export abstract class AbstractMigrationAgent<Generics extends MigrationAgentGene
         direction,
         migrations,
         state,
-        migrationContext
+        migrationContext,
+        logger
       });
 
       await this.writeLogsToStore({
         log_stream: logStream,
         state
       });
     } finally {
-      logger.info('Releasing migration lock');
+      logger.debug('Releasing migration lock');
       await releaseLock();
       process.removeListener('beforeExit', releaseLock);
-      logger.info('Done with migrations');
+      logger.debug('Done with migrations');
     }
   }
 
   protected async *execute(params: ExecuteParams): AsyncGenerator<defs.ExecutedMigration> {
+    const logger = params.logger;
     const internalMigrations = await this.loadInternalMigrations();
     let migrations = [...internalMigrations, ...params.migrations];
 

modules/module-mongodb-storage/src/utils/test-utils.ts

@@ -1,6 +1,6 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { PowerSyncMongo } from '../storage/implementation/db.js';
-import { TestStorageOptions } from '@powersync/service-core';
+import { TestStorageConfig, TestStorageOptions } from '@powersync/service-core';
 import { MongoReportStorage } from '../storage/MongoReportStorage.js';
 import { MongoBucketStorage } from '../storage/MongoBucketStorage.js';
 import { MongoSyncBucketStorageOptions } from '../storage/implementation/MongoSyncBucketStorage.js';
@@ -12,22 +12,25 @@ export type MongoTestStorageOptions = {
 };
 
 export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) {
-  return async (options?: TestStorageOptions) => {
-    const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI);
+  return {
+    factory: async (options?: TestStorageOptions) => {
+      const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI);
 
-    // None of the tests insert data into this collection, so it was never created
-    if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
-      await db.db.createCollection('bucket_parameters');
-    }
+      // None of the tests insert data into this collection, so it was never created
+      if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
+        await db.db.createCollection('bucket_parameters');
+      }
 
-    // Full migrations are not currently run for tests, so we manually create this
-    await db.createCheckpointEventsCollection();
+      // Full migrations are not currently run for tests, so we manually create this
+      await db.createCheckpointEventsCollection();
 
-    if (!options?.doNotClear) {
-      await db.clear();
-    }
+      if (!options?.doNotClear) {
+        await db.clear();
+      }
 
-    return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }, factoryOptions.internalOptions);
+      return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }, factoryOptions.internalOptions);
+    },
+    tableIdStrings: false
   };
 }
 

modules/module-mongodb-storage/test/src/__snapshots__/storage.test.ts.snap

@@ -1,11 +1,13 @@
-import { register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 import { storage, SyncRulesBucketStorage } from '@powersync/service-core';
 
 describe('Mongo Sync Bucket Storage Compact', () => {
   register.registerCompactTests(INITIALIZED_MONGO_STORAGE_FACTORY);
 
+  const TEST_TABLE = test_utils.makeTestTable('test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
+
   describe('with blank bucket_state', () => {
     // This can happen when migrating from older service versions, that did not populate bucket_state yet.
     const populate = async (bucketStorage: SyncRulesBucketStorage) => {
@@ -39,7 +41,7 @@ describe('Mongo Sync Bucket Storage Compact', () => {
     };
 
     const setup = async () => {
-      await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY();
+      await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
       const syncRules = await factory.updateSyncRules({
         content: `
 bucket_definitions:

modules/module-mongodb-storage/test/src/storage_compacting.test.ts

@@ -1,10 +1,11 @@
 import { storage } from '@powersync/service-core';
-import { register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 
 describe('sync - mongodb', () => {
   register.registerSyncTests(INITIALIZED_MONGO_STORAGE_FACTORY);
+  const TEST_TABLE = test_utils.makeTestTable('test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
 
   // The split of returned results can vary depending on storage drivers
   test('large batch (2)', async () => {
@@ -19,7 +20,7 @@ describe('sync - mongodb', () => {
           - SELECT id, description FROM "%"
     `
     );
-    await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY();
+    await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
     const bucketStorage = factory.getInstance(sync_rules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {

modules/module-mongodb-storage/test/src/storage_sync.test.ts

@@ -21,7 +21,9 @@ describe('change stream', () => {
   describeWithStorage({ timeout: 20_000 }, defineChangeStreamTests);
 });
 
-function defineChangeStreamTests(factory: storage.TestStorageFactory) {
+function defineChangeStreamTests(config: storage.TestStorageConfig) {
+  const factory = config.factory;
+
   test('replicating basic values', async () => {
     await using context = await ChangeStreamTestContext.open(factory, {
       mongoOptions: { postImages: PostImagesOption.READ_ONLY }

modules/module-mongodb/test/src/change_stream.test.ts

@@ -1,5 +1,5 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { reduceBucket, TestStorageFactory } from '@powersync/service-core';
+import { reduceBucket, TestStorageConfig, TestStorageFactory } from '@powersync/service-core';
 import { METRICS_HELPER } from '@powersync/service-core-tests';
 import { JSONBig } from '@powersync/service-jsonbig';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
@@ -12,7 +12,9 @@ describe('chunked snapshots', () => {
   describeWithStorage({ timeout: 120_000 }, defineBatchTests);
 });
 
-function defineBatchTests(factory: TestStorageFactory) {
+function defineBatchTests(config: TestStorageConfig) {
+  const { factory } = config;
+
   // This is not as sensitive to the id type as postgres, but we still test a couple of cases
   test('chunked snapshot (int32)', async () => {
     await testChunkedSnapshot({

modules/module-mongodb/test/src/chunked_snapshot.test.ts

@@ -1,7 +1,7 @@
 import { ChangeStreamInvalidatedError } from '@module/replication/ChangeStream.js';
 import { MongoManager } from '@module/replication/MongoManager.js';
 import { normalizeConnectionConfig } from '@module/types/types.js';
-import { BucketStorageFactory, TestStorageOptions } from '@powersync/service-core';
+import { TestStorageConfig } from '@powersync/service-core';
 import { describe, expect, test } from 'vitest';
 import { ChangeStreamTestContext } from './change_stream_utils.js';
 import { env } from './env.js';
@@ -11,7 +11,9 @@ describe('mongodb resuming replication', () => {
   describeWithStorage({}, defineResumeTest);
 });
 
-function defineResumeTest(factoryGenerator: (options?: TestStorageOptions) => Promise<BucketStorageFactory>) {
+function defineResumeTest(config: TestStorageConfig) {
+  const factoryGenerator = config.factory;
+
   test('resuming with a different source database', async () => {
     await using context = await ChangeStreamTestContext.open(factoryGenerator);
     const { db } = context;

modules/module-mongodb/test/src/resume.test.ts

@@ -8,14 +8,14 @@ import { env } from './env.js';
 import { describeWithStorage } from './util.js';
 
 describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () {
-  describeWithStorage({ timeout: 240_000 }, function (factory) {
+  describeWithStorage({ timeout: 240_000 }, function (config) {
     test('resuming initial replication (1)', async () => {
       // Stop early - likely to not include deleted row in first replication attempt.
-      await testResumingReplication(factory, 2000);
+      await testResumingReplication(config.factory, 2000);
     });
     test('resuming initial replication (2)', async () => {
       // Stop late - likely to include deleted row in first replication attempt.
-      await testResumingReplication(factory, 8000);
+      await testResumingReplication(config.factory, 8000);
     });
   });
 });