Restructure TEST_TABLE for tests.

Author: rkistner

libs/lib-services/src/logger/logger-index.ts

@@ -1,2 +1,3 @@
 export * from './Logger.js';
 export { Logger } from 'winston';
+export { createLogger, format, transports } from 'winston';

libs/lib-services/src/migrations/AbstractMigrationAgent.ts

@@ -1,11 +1,12 @@
 import { LockManager } from '../locks/LockManager.js';
-import { logger } from '../logger/Logger.js';
+import { logger as defaultLogger, Logger } from '../logger/logger-index.js';
 import * as defs from './migration-definitions.js';
 
 export type MigrationParams<Generics extends MigrationAgentGenerics = MigrationAgentGenerics> = {
   count?: number;
   direction: defs.Direction;
   migrationContext?: Generics['MIGRATION_CONTEXT'];
+  logger?: Logger;
 };
 
 type WriteLogsParams = {
@@ -20,10 +21,12 @@ export type MigrationAgentGenerics = {
 export type RunMigrationParams<Generics extends MigrationAgentGenerics = MigrationAgentGenerics> = MigrationParams & {
   migrations: defs.Migration<Generics['MIGRATION_CONTEXT']>[];
   maxLockWaitMs?: number;
+  logger?: Logger;
 };
 
 type ExecuteParams = RunMigrationParams & {
   state?: defs.MigrationState;
+  logger: Logger;
 };
 
 export const DEFAULT_MAX_LOCK_WAIT_MS = 3 * 60 * 1000; // 3 minutes
@@ -46,9 +49,11 @@ export abstract class AbstractMigrationAgent<Generics extends MigrationAgentGene
   async run(params: RunMigrationParams) {
     await this.init();
 
+    const logger = params.logger ?? defaultLogger;
+
     const { direction, migrations, migrationContext } = params;
     // Only one process should execute this at a time.
-    logger.info('Acquiring lock for migrations');
+    logger.debug('Acquiring lock for migrations');
     const lockHandle = await this.locks.acquire({ max_wait_ms: params.maxLockWaitMs ?? DEFAULT_MAX_LOCK_WAIT_MS });
 
     if (!lockHandle) {
@@ -75,22 +80,24 @@ export abstract class AbstractMigrationAgent<Generics extends MigrationAgentGene
         direction,
         migrations,
         state,
-        migrationContext
+        migrationContext,
+        logger
       });
 
       await this.writeLogsToStore({
         log_stream: logStream,
         state
       });
     } finally {
-      logger.info('Releasing migration lock');
+      logger.debug('Releasing migration lock');
       await releaseLock();
       process.removeListener('beforeExit', releaseLock);
-      logger.info('Done with migrations');
+      logger.debug('Done with migrations');
     }
   }
 
   protected async *execute(params: ExecuteParams): AsyncGenerator<defs.ExecutedMigration> {
+    const logger = params.logger;
     const internalMigrations = await this.loadInternalMigrations();
     let migrations = [...internalMigrations, ...params.migrations];
 

modules/module-mongodb-storage/src/utils/test-utils.ts

@@ -1,6 +1,6 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { PowerSyncMongo } from '../storage/implementation/db.js';
-import { TestStorageOptions } from '@powersync/service-core';
+import { TestStorageConfig, TestStorageOptions } from '@powersync/service-core';
 import { MongoReportStorage } from '../storage/MongoReportStorage.js';
 import { MongoBucketStorage } from '../storage/MongoBucketStorage.js';
 import { MongoSyncBucketStorageOptions } from '../storage/implementation/MongoSyncBucketStorage.js';
@@ -12,22 +12,25 @@ export type MongoTestStorageOptions = {
 };
 
 export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) {
-  return async (options?: TestStorageOptions) => {
-    const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI);
+  return {
+    factory: async (options?: TestStorageOptions) => {
+      const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI);
 
-    // None of the tests insert data into this collection, so it was never created
-    if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
-      await db.db.createCollection('bucket_parameters');
-    }
+      // None of the tests insert data into this collection, so it was never created
+      if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
+        await db.db.createCollection('bucket_parameters');
+      }
 
-    // Full migrations are not currently run for tests, so we manually create this
-    await db.createCheckpointEventsCollection();
+      // Full migrations are not currently run for tests, so we manually create this
+      await db.createCheckpointEventsCollection();
 
-    if (!options?.doNotClear) {
-      await db.clear();
-    }
+      if (!options?.doNotClear) {
+        await db.clear();
+      }
 
-    return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }, factoryOptions.internalOptions);
+      return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }, factoryOptions.internalOptions);
+    },
+    tableIdStrings: false
   };
 }
 

modules/module-mongodb-storage/test/src/__snapshots__/storage.test.ts.snap

@@ -1,10 +1,11 @@
 import { storage } from '@powersync/service-core';
-import { register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 
 describe('sync - mongodb', () => {
   register.registerSyncTests(INITIALIZED_MONGO_STORAGE_FACTORY);
+  const TEST_TABLE = test_utils.makeTestTable('test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
 
   // The split of returned results can vary depending on storage drivers
   test('large batch (2)', async () => {
@@ -19,7 +20,7 @@ describe('sync - mongodb', () => {
           - SELECT id, description FROM "%"
     `
     );
-    await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY();
+    await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
     const bucketStorage = factory.getInstance(sync_rules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {

modules/module-mongodb-storage/test/src/storage_sync.test.ts

@@ -3,6 +3,7 @@ import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'
 import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js';
 import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
 import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js';
+import { logger as defaultLogger, createLogger, transports } from '@powersync/lib-services-framework';
 
 export type PostgresTestStorageOptions = {
   url: string;
@@ -31,19 +32,28 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
 
     const mockServiceContext = { configuration: { storage: BASE_CONFIG } } as unknown as ServiceContext;
 
+    // Migration logs can get really verbose in tests, so only log warnings and up.
+    const logger = createLogger({
+      level: 'warn',
+      format: defaultLogger.format,
+      transports: [new transports.Console()]
+    });
+
     await migrationManager.migrate({
       direction: framework.migrations.Direction.Down,
       migrationContext: {
         service_context: mockServiceContext
-      }
+      },
+      logger
     });
 
     if (direction == framework.migrations.Direction.Up) {
       await migrationManager.migrate({
         direction: framework.migrations.Direction.Up,
         migrationContext: {
           service_context: mockServiceContext
-        }
+        },
+        logger
       });
     }
   };
@@ -80,7 +90,8 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
         throw ex;
       }
     },
-    migrate
+    migrate,
+    tableIdStrings: true
   };
 }
 

modules/module-postgres-storage/src/utils/test-utils.ts

@@ -0,0 +1,94 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Postgres Sync Bucket Storage Compact > compacting (1) 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [],
+      "last_op_id": "0",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "by_user",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "0",
+    },
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage Compact > compacting (1) 2`] = `
+[
+  {
+    "checkpoint_diff": {
+      "last_op_id": "1",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "by_user["user1"]",
+          "checksum": 0,
+          "count": 0,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage Compact > compacting (1) 3`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": -93886621,
+          "count": 2,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "2",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "mybucket",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage Compact > compacting (2) 1`] = `
+[
+  {
+    "token_expires_in": 0,
+  },
+]
+`;

modules/module-postgres-storage/test/src/__snapshots__/storage.test.ts.snap

@@ -21,7 +21,7 @@ describe('Migrations', () => {
   register.registerMigrationTests(MIGRATION_AGENT_FACTORY);
 
   it('Should have tables declared', async () => {
-    const { db } = await POSTGRES_STORAGE_FACTORY();
+    const { db } = await POSTGRES_STORAGE_FACTORY.factory();
 
     const tables = await db.sql`
       SELECT

modules/module-postgres-storage/test/src/__snapshots__/storage_compacting.test.ts.snap

@@ -1,5 +1,5 @@
 import { storage } from '@powersync/service-core';
-import { register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import { POSTGRES_STORAGE_FACTORY } from './util.js';
 
@@ -32,11 +32,11 @@ describe('Postgres Sync Bucket Storage - pg-specific', () => {
           - SELECT id, description FROM "%"
     `
     );
-    await using factory = await POSTGRES_STORAGE_FACTORY();
+    await using factory = await POSTGRES_STORAGE_FACTORY.factory();
     const bucketStorage = factory.getInstance(sync_rules);
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-      const sourceTable = TEST_TABLE;
+      const sourceTable = test_utils.makeTestTable('test', ['id'], POSTGRES_STORAGE_FACTORY);
 
       const largeDescription = '0123456789'.repeat(2_000_00);