From ee174ae1ab3917286d68d5df5078506bc79d951d Mon Sep 17 00:00:00 2001 From: Najam Ahmed Ansari Date: Fri, 28 Nov 2025 00:22:59 +0500 Subject: [PATCH 1/7] Dynamically load connection modules for reduced memory usage Done in response to #410. Refactors module loading to dynamically import database connection and storage modules (MySQL, PostgreSQL) based on the configuration, rather than registering them all upfront. Signed-off-by: Najam Ahmed Ansari --- .changeset/perfect-plants-report.md | 5 ++ service/package.json | 3 +- service/src/entry.ts | 9 +-- service/src/runners/server.ts | 10 ++- service/src/runners/stream-worker.ts | 11 ++- service/src/runners/unified-runner.ts | 10 ++- service/src/util/module-loader.ts | 56 ++++++++++++++ service/test/src/util/module-loader.test.ts | 85 +++++++++++++++++++++ 8 files changed, 175 insertions(+), 14 deletions(-) create mode 100644 .changeset/perfect-plants-report.md create mode 100644 service/src/util/module-loader.ts create mode 100644 service/test/src/util/module-loader.test.ts diff --git a/.changeset/perfect-plants-report.md b/.changeset/perfect-plants-report.md new file mode 100644 index 000000000..ac33e340e --- /dev/null +++ b/.changeset/perfect-plants-report.md @@ -0,0 +1,5 @@ +--- +'@powersync/service-image': minor +--- + +Dynamically load connection modules for reduced memory usage diff --git a/service/package.json b/service/package.json index 15ebfa29c..3abd823b2 100644 --- a/service/package.json +++ b/service/package.json @@ -7,7 +7,8 @@ "scripts": { "build": "tsc -b", "watch": "nodemon -w ../ -e ts -e js --delay 1 -x node --loader ts-node/esm src/entry.ts start", - "clean": "rm -rf ./lib && tsc -b --clean" + "clean": "rm -rf ./lib && tsc -b --clean", + "test": "vitest" }, "dependencies": { "@powersync/service-core": "workspace:*", diff --git a/service/src/entry.ts b/service/src/entry.ts index 61b943f17..18ed1aefe 100644 --- a/service/src/entry.ts +++ b/service/src/entry.ts @@ -17,14 +17,7 @@ container.registerDefaults(); container.register(ContainerImplementation.REPORTER, createSentryReporter()); const moduleManager = new core.modules.ModuleManager(); -moduleManager.register([ - new CoreModule(), - new MongoModule(), - new MongoStorageModule(), - new MySQLModule(), - new PostgresModule(), - new PostgresStorageModule() -]); +moduleManager.register([new CoreModule(), new MongoModule(), new MongoStorageModule()]); // This is a bit of a hack. Commands such as the teardown command or even migrations might // want access to the ModuleManager in order to use modules container.register(core.ModuleManager, moduleManager); diff --git a/service/src/runners/server.ts b/service/src/runners/server.ts index 5876a6977..92c092fc4 100644 --- a/service/src/runners/server.ts +++ b/service/src/runners/server.ts @@ -1,5 +1,7 @@ import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; + +import { loadModules } from '../util/module-loader.js'; import { logBooting } from '../util/version.js'; /** @@ -9,12 +11,18 @@ export async function startServer(runnerConfig: core.utils.RunnerConfig) { logBooting('API Container'); const config = await core.utils.loadConfig(runnerConfig); + + const moduleManager = container.getImplementation(core.modules.ModuleManager); + const modules = await loadModules(config); + if (modules.length > 0) { + moduleManager.register(modules); + } + const serviceContext = new core.system.ServiceContextContainer({ serviceMode: core.system.ServiceContextMode.API, configuration: config }); - const moduleManager = container.getImplementation(core.modules.ModuleManager); await moduleManager.initialize(serviceContext); logger.info('Starting service...'); diff --git a/service/src/runners/stream-worker.ts b/service/src/runners/stream-worker.ts index e9197a376..9134fa549 100644 --- a/service/src/runners/stream-worker.ts +++ b/service/src/runners/stream-worker.ts @@ -1,5 +1,7 @@ import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; + +import { loadModules } from '../util/module-loader.js'; import { logBooting } from '../util/version.js'; /** @@ -20,15 +22,20 @@ export const startStreamRunner = async (runnerConfig: core.utils.RunnerConfig) = logBooting('Replication Container'); const config = await core.utils.loadConfig(runnerConfig); + + const moduleManager = container.getImplementation(core.modules.ModuleManager); + const modules = await loadModules(config); + if (modules.length > 0) { + moduleManager.register(modules); + } + // Self-hosted version allows for automatic migrations const serviceContext = new core.system.ServiceContextContainer({ serviceMode: core.system.ServiceContextMode.SYNC, configuration: config }); - registerReplicationServices(serviceContext); - const moduleManager = container.getImplementation(core.modules.ModuleManager); await moduleManager.initialize(serviceContext); // Ensure automatic migrations diff --git a/service/src/runners/unified-runner.ts b/service/src/runners/unified-runner.ts index c008286c9..648aaf98f 100644 --- a/service/src/runners/unified-runner.ts +++ b/service/src/runners/unified-runner.ts @@ -1,6 +1,7 @@ import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; +import { loadModules } from '../util/module-loader.js'; import { logBooting } from '../util/version.js'; import { registerReplicationServices } from './stream-worker.js'; @@ -11,14 +12,19 @@ export const startUnifiedRunner = async (runnerConfig: core.utils.RunnerConfig) logBooting('Unified Container'); const config = await core.utils.loadConfig(runnerConfig); + + const moduleManager = container.getImplementation(core.modules.ModuleManager); + const modules = await loadModules(config); + if (modules.length > 0) { + moduleManager.register(modules); + } + const serviceContext = new core.system.ServiceContextContainer({ serviceMode: core.system.ServiceContextMode.UNIFIED, configuration: config }); - registerReplicationServices(serviceContext); - const moduleManager = container.getImplementation(core.modules.ModuleManager); await moduleManager.initialize(serviceContext); await core.migrations.ensureAutomaticMigrations({ diff --git a/service/src/util/module-loader.ts b/service/src/util/module-loader.ts new file mode 100644 index 000000000..783886237 --- /dev/null +++ b/service/src/util/module-loader.ts @@ -0,0 +1,56 @@ +import { container, logger } from '@powersync/lib-services-framework'; +import * as core from '@powersync/service-core'; + +interface DynamicModuleMap { + [key: string]: () => Promise; +} + +const ModuleMap: DynamicModuleMap = { + mysql: () => import('@powersync/service-module-mysql').then((module) => module.MySQLModule), + postgresql: () => import('@powersync/service-module-postgres').then((module) => module.PostgresModule), + 'postgresql-storage': () => + import('@powersync/service-module-postgres-storage').then((module) => module.PostgresStorageModule) +}; + +/** + * Utility function to dynamically load and instantiate modules. + * This function can optionally be moved to its own file (e.g., module-loader.ts) + * for better separation of concerns if the file gets much larger. + */ +export async function loadModules(config: core.ResolvedPowerSyncConfig) { + // 1. Determine required connections: Unique types from connections + storage type + const requiredConnections = [ + ...new Set(config.connections?.map((connection) => connection.type) || []), + `${config.storage.type}-storage` // Using template literal is clear + ]; + + // 2. Map connection types to their module loading promises + const modulePromises = requiredConnections.map(async (connectionType) => { + // Exclude 'mongo' connections explicitly early + if (connectionType.startsWith('mongo')) { + return null; // Return null for filtering later + } + + const modulePromise = ModuleMap[connectionType]; + + // Check if a module is defined for the connection type + if (!modulePromise) { + logger.warn(`No module defined in ModuleMap for connection type: ${connectionType}`); + return null; + } + + try { + // Dynamically import and instantiate the class + const ModuleClass = await modulePromise(); + return new ModuleClass(); + } catch (error) { + // Log an error if the dynamic import fails (e.g., module not installed) + logger.error(`Failed to load module for ${connectionType}:`, error); + return null; + } + }); + + // 3. Resolve all promises and filter out nulls/undefineds + const moduleInstances = await Promise.all(modulePromises); + return moduleInstances.filter((instance) => instance !== null); // Filter out nulls from excluded or failed imports +} diff --git a/service/test/src/util/module-loader.test.ts b/service/test/src/util/module-loader.test.ts new file mode 100644 index 000000000..21c1a17c1 --- /dev/null +++ b/service/test/src/util/module-loader.test.ts @@ -0,0 +1,85 @@ +import { describe, test, expect, it } from 'vitest'; + +import { logger } from '@powersync/lib-services-framework'; + +import { MySQLModule } from '@powersync/service-module-mysql'; +import { PostgresModule } from '@powersync/service-module-postgres'; +import { PostgresStorageModule } from '@powersync/service-module-postgres-storage'; +import { loadModules } from '../../../src/util/module-loader.js'; + +interface MockConfig { + connections?: MockConnection[]; + storage: { type: string }; +} + +describe('module loader', () => { + it('should load all modules defined in connections and storage', async () => { + const config: MockConfig = { + connections: [{ type: 'mysql' }, { type: 'postgresql' }], + storage: { type: 'postgresql' } // This should result in 'postgresql-storage' + }; + + const modules = await loadModules(config); + + expect(modules.length).toBe(3); + expect(modules[0]).toBeInstanceOf(MySQLModule); + expect(modules[1]).toBeInstanceOf(PostgresModule); + expect(modules[2]).toBeInstanceOf(PostgresStorageModule); + }); + + it('should handle duplicate connection types (e.g., mysql used twice)', async () => { + const config: MockConfig = { + connections: [{ type: 'mysql' }, { type: 'postgresql' }, { type: 'mysql' }], // mysql duplicated + storage: { type: 'postgresql' } + }; + + const modules = await loadModules(config); + + // Expect 3 modules: mysql, postgresql, postgresql-storage + expect(modules.length).toBe(3); + expect(modules.filter((m) => m instanceof MySQLModule).length).toBe(1); + expect(modules.filter((m) => m instanceof PostgresModule).length).toBe(1); + expect(modules.filter((m) => m instanceof PostgresStorageModule).length).toBe(1); + }); + + it('should exclude connections starting with "mongo"', async () => { + const config: MockConfig = { + connections: [{ type: 'mysql' }, { type: 'mongodb' }], // mongodb should be ignored + storage: { type: 'postgresql' } + }; + + const modules = await loadModules(config); + + // Expect 2 modules: mysql and postgresql-storage + expect(modules.length).toBe(2); + expect(modules[0]).toBeInstanceOf(MySQLModule); + expect(modules[1]).toBeInstanceOf(PostgresStorageModule); + expect(modules.filter((m) => m instanceof PostgresModule).length).toBe(0); + }); + + it('should filter out modules not found in ModuleMap', async () => { + const config: MockConfig = { + connections: [{ type: 'mysql' }, { type: 'redis' }], // unknown-db is missing + storage: { type: 'postgresql' } + }; + + const modules = await loadModules(config); + + // Expect 2 modules: mysql and postgresql-storage + expect(modules.length).toBe(2); + expect(modules.every((m) => m instanceof MySQLModule || m instanceof PostgresStorageModule)).toBe(true); + }); + + it('should filter out modules that fail to import and log an error', async () => { + const config: MockConfig = { + connections: [{ type: 'mysql' }, { type: 'failing-module' }], // failing-module rejects promise + storage: { type: 'postgresql' } + }; + + const modules = await loadModules(config); + + // Expect 2 modules: mysql and postgresql-storage + expect(modules.length).toBe(2); + expect(modules.filter((m) => m instanceof MySQLModule).length).toBe(1); + }); +}); From a9b9e8d21e97410e8d7263c27163345272f857e2 Mon Sep 17 00:00:00 2001 From: Najam Ahmed Ansari Date: Sat, 29 Nov 2025 18:36:50 +0500 Subject: [PATCH 2/7] Incorporating PR feedback (cleanups) --- service/src/entry.ts | 3 --- service/src/util/module-loader.ts | 4 +--- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/service/src/entry.ts b/service/src/entry.ts index 18ed1aefe..a1ee6b79a 100644 --- a/service/src/entry.ts +++ b/service/src/entry.ts @@ -4,9 +4,6 @@ import * as core from '@powersync/service-core'; import { CoreModule } from '@powersync/service-module-core'; import { MongoModule } from '@powersync/service-module-mongodb'; import { MongoStorageModule } from '@powersync/service-module-mongodb-storage'; -import { MySQLModule } from '@powersync/service-module-mysql'; -import { PostgresModule } from '@powersync/service-module-postgres'; -import { PostgresStorageModule } from '@powersync/service-module-postgres-storage'; import { startServer } from './runners/server.js'; import { startStreamRunner } from './runners/stream-worker.js'; import { startUnifiedRunner } from './runners/unified-runner.js'; diff --git a/service/src/util/module-loader.ts b/service/src/util/module-loader.ts index 783886237..18074a194 100644 --- a/service/src/util/module-loader.ts +++ b/service/src/util/module-loader.ts @@ -14,8 +14,6 @@ const ModuleMap: DynamicModuleMap = { /** * Utility function to dynamically load and instantiate modules. - * This function can optionally be moved to its own file (e.g., module-loader.ts) - * for better separation of concerns if the file gets much larger. */ export async function loadModules(config: core.ResolvedPowerSyncConfig) { // 1. Determine required connections: Unique types from connections + storage type @@ -46,7 +44,7 @@ export async function loadModules(config: core.ResolvedPowerSyncConfig) { } catch (error) { // Log an error if the dynamic import fails (e.g., module not installed) logger.error(`Failed to load module for ${connectionType}:`, error); - return null; + throw error; } }); From b209cbe4b0fd8e9b7312f2109e6136afb2bc0b42 Mon Sep 17 00:00:00 2001 From: Najam Ahmed Ansari Date: Sat, 29 Nov 2025 23:36:28 +0500 Subject: [PATCH 3/7] Refactoring module loader and its test cases - Split modules between storage and connection. - Cleaner code that now checks if given connection/storage types can be handled by us or not. - Error raised if any connection types cannot be handled or if any imports fail. - Explicit test case that simulates import failure for a module. Signed-off-by: Najam Ahmed Ansari --- service/src/util/module-loader.ts | 69 ++++++++++++--------- service/test/src/util/module-loader.test.ts | 34 +++++----- 2 files changed, 57 insertions(+), 46 deletions(-) diff --git a/service/src/util/module-loader.ts b/service/src/util/module-loader.ts index 18074a194..0ceccd9ba 100644 --- a/service/src/util/module-loader.ts +++ b/service/src/util/module-loader.ts @@ -5,50 +5,57 @@ interface DynamicModuleMap { [key: string]: () => Promise; } -const ModuleMap: DynamicModuleMap = { +export const ConnectionModuleMap: DynamicModuleMap = { mysql: () => import('@powersync/service-module-mysql').then((module) => module.MySQLModule), - postgresql: () => import('@powersync/service-module-postgres').then((module) => module.PostgresModule), - 'postgresql-storage': () => - import('@powersync/service-module-postgres-storage').then((module) => module.PostgresStorageModule) + postgresql: () => import('@powersync/service-module-postgres').then((module) => module.PostgresModule) +}; + +const StorageModuleMap: DynamicModuleMap = { + postgresql: () => import('@powersync/service-module-postgres-storage').then((module) => module.PostgresStorageModule) }; /** * Utility function to dynamically load and instantiate modules. */ export async function loadModules(config: core.ResolvedPowerSyncConfig) { - // 1. Determine required connections: Unique types from connections + storage type const requiredConnections = [ - ...new Set(config.connections?.map((connection) => connection.type) || []), - `${config.storage.type}-storage` // Using template literal is clear + ...new Set( + config.connections + ?.map((connection) => connection.type) + .filter((connection) => !connection.startsWith('mongo')) || [] + ) ]; - - // 2. Map connection types to their module loading promises - const modulePromises = requiredConnections.map(async (connectionType) => { - // Exclude 'mongo' connections explicitly early - if (connectionType.startsWith('mongo')) { - return null; // Return null for filtering later + const missingConnectionModules: string[] = []; + const modulePromises: Promise[] = []; + + // 1. Map connection types to their module loading promises making note of any + // missing connection types. + requiredConnections.forEach((connectionType) => { + const modulePromise = ConnectionModuleMap[connectionType]; + if (modulePromise !== undefined) { + modulePromises.push(modulePromise()); + } else { + missingConnectionModules.push(connectionType); } + }); - const modulePromise = ModuleMap[connectionType]; + // Fail if any connection types are not found. + if (missingConnectionModules.length > 0) { + throw new Error(`Invalid connection types: "${[...missingConnectionModules].join(', ')}"`); + } - // Check if a module is defined for the connection type - if (!modulePromise) { - logger.warn(`No module defined in ModuleMap for connection type: ${connectionType}`); - return null; - } + if (config.storage.type !== 'mongo' && StorageModuleMap[config.storage.type] !== undefined) { + modulePromises.push(StorageModuleMap[config.storage.type]()); + } - try { - // Dynamically import and instantiate the class - const ModuleClass = await modulePromise(); + // 2. Dynamically import and instantiate module classes and resolve all promises + // raising errors if any modules could not be imported. + const moduleInstances = await Promise.all( + modulePromises.map(async (modulePromise) => { + const ModuleClass = await modulePromise; return new ModuleClass(); - } catch (error) { - // Log an error if the dynamic import fails (e.g., module not installed) - logger.error(`Failed to load module for ${connectionType}:`, error); - throw error; - } - }); + }) + ); - // 3. Resolve all promises and filter out nulls/undefineds - const moduleInstances = await Promise.all(modulePromises); - return moduleInstances.filter((instance) => instance !== null); // Filter out nulls from excluded or failed imports + return moduleInstances; } diff --git a/service/test/src/util/module-loader.test.ts b/service/test/src/util/module-loader.test.ts index 21c1a17c1..d1713bc5b 100644 --- a/service/test/src/util/module-loader.test.ts +++ b/service/test/src/util/module-loader.test.ts @@ -1,4 +1,4 @@ -import { describe, test, expect, it } from 'vitest'; +import { vi, describe, test, expect, it } from 'vitest'; import { logger } from '@powersync/lib-services-framework'; @@ -18,7 +18,6 @@ describe('module loader', () => { connections: [{ type: 'mysql' }, { type: 'postgresql' }], storage: { type: 'postgresql' } // This should result in 'postgresql-storage' }; - const modules = await loadModules(config); expect(modules.length).toBe(3); @@ -57,29 +56,34 @@ describe('module loader', () => { expect(modules.filter((m) => m instanceof PostgresModule).length).toBe(0); }); - it('should filter out modules not found in ModuleMap', async () => { + it('should throw an error if any modules are not found in ModuleMap', async () => { const config: MockConfig = { connections: [{ type: 'mysql' }, { type: 'redis' }], // unknown-db is missing storage: { type: 'postgresql' } }; - const modules = await loadModules(config); - - // Expect 2 modules: mysql and postgresql-storage - expect(modules.length).toBe(2); - expect(modules.every((m) => m instanceof MySQLModule || m instanceof PostgresStorageModule)).toBe(true); + await expect(loadModules(config)).rejects.toThrowError(); }); - it('should filter out modules that fail to import and log an error', async () => { + it('should throw an error if one dynamic connection module import fails', async () => { + vi.doMock('../../../src/util/module-loader.js', async (importOriginal) => { + const mod = await importOriginal(); + mod.ConnectionModuleMap.mysql = () => + import('@powersync/service-module-mysql').then(() => { + throw new Error('Failed to load MySQL module'); + }); + return mod; + }); + + const { loadModules } = await import('../../../src/util/module-loader.js'); + const config: MockConfig = { - connections: [{ type: 'mysql' }, { type: 'failing-module' }], // failing-module rejects promise - storage: { type: 'postgresql' } + connections: [{ type: 'mysql' }], + storage: { type: 'mongodb' } }; - const modules = await loadModules(config); + await expect(loadModules(config)).rejects.toThrowError('Failed to load MySQL module'); - // Expect 2 modules: mysql and postgresql-storage - expect(modules.length).toBe(2); - expect(modules.filter((m) => m instanceof MySQLModule).length).toBe(1); + vi.doUnmock('@powersync/service-module-mysql'); }); }); From 24950bb5374c04d270e72e5b613b4fe8f78fee91 Mon Sep 17 00:00:00 2001 From: Najam Ahmed Ansari Date: Mon, 1 Dec 2025 17:02:33 +0500 Subject: [PATCH 4/7] Removing MongoDB exclusions Signed-off-by: Najam Ahmed Ansari --- service/src/entry.ts | 4 +--- service/src/util/module-loader.ts | 7 ++++--- service/test/src/util/module-loader.test.ts | 19 ++----------------- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/service/src/entry.ts b/service/src/entry.ts index a1ee6b79a..9d2bac42a 100644 --- a/service/src/entry.ts +++ b/service/src/entry.ts @@ -2,8 +2,6 @@ import { container, ContainerImplementation } from '@powersync/lib-services-fram import * as core from '@powersync/service-core'; import { CoreModule } from '@powersync/service-module-core'; -import { MongoModule } from '@powersync/service-module-mongodb'; -import { MongoStorageModule } from '@powersync/service-module-mongodb-storage'; import { startServer } from './runners/server.js'; import { startStreamRunner } from './runners/stream-worker.js'; import { startUnifiedRunner } from './runners/unified-runner.js'; @@ -14,7 +12,7 @@ container.registerDefaults(); container.register(ContainerImplementation.REPORTER, createSentryReporter()); const moduleManager = new core.modules.ModuleManager(); -moduleManager.register([new CoreModule(), new MongoModule(), new MongoStorageModule()]); +moduleManager.register([new CoreModule()]); // This is a bit of a hack. Commands such as the teardown command or even migrations might // want access to the ModuleManager in order to use modules container.register(core.ModuleManager, moduleManager); diff --git a/service/src/util/module-loader.ts b/service/src/util/module-loader.ts index 0ceccd9ba..bd138e668 100644 --- a/service/src/util/module-loader.ts +++ b/service/src/util/module-loader.ts @@ -6,11 +6,13 @@ interface DynamicModuleMap { } export const ConnectionModuleMap: DynamicModuleMap = { + mongodb: () => import('@powersync/service-module-mongodb').then((module) => module.MongoModule), mysql: () => import('@powersync/service-module-mysql').then((module) => module.MySQLModule), postgresql: () => import('@powersync/service-module-postgres').then((module) => module.PostgresModule) }; const StorageModuleMap: DynamicModuleMap = { + mongodb: () => import('@powersync/service-module-mongodb-storage').then((module) => module.MongoStorageModule), postgresql: () => import('@powersync/service-module-postgres-storage').then((module) => module.PostgresStorageModule) }; @@ -21,8 +23,7 @@ export async function loadModules(config: core.ResolvedPowerSyncConfig) { const requiredConnections = [ ...new Set( config.connections - ?.map((connection) => connection.type) - .filter((connection) => !connection.startsWith('mongo')) || [] + ?.map((connection) => connection.type) || [] ) ]; const missingConnectionModules: string[] = []; @@ -44,7 +45,7 @@ export async function loadModules(config: core.ResolvedPowerSyncConfig) { throw new Error(`Invalid connection types: "${[...missingConnectionModules].join(', ')}"`); } - if (config.storage.type !== 'mongo' && StorageModuleMap[config.storage.type] !== undefined) { + if (StorageModuleMap[config.storage.type] !== undefined) { modulePromises.push(StorageModuleMap[config.storage.type]()); } diff --git a/service/test/src/util/module-loader.test.ts b/service/test/src/util/module-loader.test.ts index d1713bc5b..3b16ad3be 100644 --- a/service/test/src/util/module-loader.test.ts +++ b/service/test/src/util/module-loader.test.ts @@ -16,7 +16,7 @@ describe('module loader', () => { it('should load all modules defined in connections and storage', async () => { const config: MockConfig = { connections: [{ type: 'mysql' }, { type: 'postgresql' }], - storage: { type: 'postgresql' } // This should result in 'postgresql-storage' + storage: { type: 'postgresql' } }; const modules = await loadModules(config); @@ -41,24 +41,9 @@ describe('module loader', () => { expect(modules.filter((m) => m instanceof PostgresStorageModule).length).toBe(1); }); - it('should exclude connections starting with "mongo"', async () => { - const config: MockConfig = { - connections: [{ type: 'mysql' }, { type: 'mongodb' }], // mongodb should be ignored - storage: { type: 'postgresql' } - }; - - const modules = await loadModules(config); - - // Expect 2 modules: mysql and postgresql-storage - expect(modules.length).toBe(2); - expect(modules[0]).toBeInstanceOf(MySQLModule); - expect(modules[1]).toBeInstanceOf(PostgresStorageModule); - expect(modules.filter((m) => m instanceof PostgresModule).length).toBe(0); - }); - it('should throw an error if any modules are not found in ModuleMap', async () => { const config: MockConfig = { - connections: [{ type: 'mysql' }, { type: 'redis' }], // unknown-db is missing + connections: [{ type: 'mysql' }, { type: 'redis' }], storage: { type: 'postgresql' } }; From 2a7df8b2defa107af91054822ed3dea43ae8ad10 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 4 Dec 2025 10:26:25 +0200 Subject: [PATCH 5/7] Some cleanup. --- service/src/util/module-loader.ts | 42 +++++++++------------ service/test/src/util/module-loader.test.ts | 27 ++++++------- 2 files changed, 31 insertions(+), 38 deletions(-) diff --git a/service/src/util/module-loader.ts b/service/src/util/module-loader.ts index bd138e668..6b245b7ce 100644 --- a/service/src/util/module-loader.ts +++ b/service/src/util/module-loader.ts @@ -1,38 +1,33 @@ -import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; interface DynamicModuleMap { - [key: string]: () => Promise; + [key: string]: () => Promise; } -export const ConnectionModuleMap: DynamicModuleMap = { - mongodb: () => import('@powersync/service-module-mongodb').then((module) => module.MongoModule), - mysql: () => import('@powersync/service-module-mysql').then((module) => module.MySQLModule), - postgresql: () => import('@powersync/service-module-postgres').then((module) => module.PostgresModule) +export const CONNECTION_MODULE_MAP: DynamicModuleMap = { + mongodb: () => import('@powersync/service-module-mongodb').then((module) => new module.MongoModule()), + mysql: () => import('@powersync/service-module-mysql').then((module) => new module.MySQLModule()), + postgresql: () => import('@powersync/service-module-postgres').then((module) => new module.PostgresModule()) }; -const StorageModuleMap: DynamicModuleMap = { - mongodb: () => import('@powersync/service-module-mongodb-storage').then((module) => module.MongoStorageModule), - postgresql: () => import('@powersync/service-module-postgres-storage').then((module) => module.PostgresStorageModule) +const STORAGE_MODULE_MAP: DynamicModuleMap = { + mongodb: () => import('@powersync/service-module-mongodb-storage').then((module) => new module.MongoStorageModule()), + postgresql: () => + import('@powersync/service-module-postgres-storage').then((module) => new module.PostgresStorageModule()) }; /** * Utility function to dynamically load and instantiate modules. */ export async function loadModules(config: core.ResolvedPowerSyncConfig) { - const requiredConnections = [ - ...new Set( - config.connections - ?.map((connection) => connection.type) || [] - ) - ]; + const requiredConnections = [...new Set(config.connections?.map((connection) => connection.type) || [])]; const missingConnectionModules: string[] = []; - const modulePromises: Promise[] = []; + const modulePromises: Promise[] = []; // 1. Map connection types to their module loading promises making note of any // missing connection types. requiredConnections.forEach((connectionType) => { - const modulePromise = ConnectionModuleMap[connectionType]; + const modulePromise = CONNECTION_MODULE_MAP[connectionType]; if (modulePromise !== undefined) { modulePromises.push(modulePromise()); } else { @@ -45,18 +40,15 @@ export async function loadModules(config: core.ResolvedPowerSyncConfig) { throw new Error(`Invalid connection types: "${[...missingConnectionModules].join(', ')}"`); } - if (StorageModuleMap[config.storage.type] !== undefined) { - modulePromises.push(StorageModuleMap[config.storage.type]()); + if (STORAGE_MODULE_MAP[config.storage.type] !== undefined) { + modulePromises.push(STORAGE_MODULE_MAP[config.storage.type]()); + } else { + throw new Error(`Invalid storage type: "${config.storage.type}"`); } // 2. Dynamically import and instantiate module classes and resolve all promises // raising errors if any modules could not be imported. - const moduleInstances = await Promise.all( - modulePromises.map(async (modulePromise) => { - const ModuleClass = await modulePromise; - return new ModuleClass(); - }) - ); + const moduleInstances = await Promise.all(modulePromises); return moduleInstances; } diff --git a/service/test/src/util/module-loader.test.ts b/service/test/src/util/module-loader.test.ts index 3b16ad3be..bdb2d6adc 100644 --- a/service/test/src/util/module-loader.test.ts +++ b/service/test/src/util/module-loader.test.ts @@ -8,7 +8,7 @@ import { PostgresStorageModule } from '@powersync/service-module-postgres-storag import { loadModules } from '../../../src/util/module-loader.js'; interface MockConfig { - connections?: MockConnection[]; + connections?: { type: string }[]; storage: { type: string }; } @@ -18,7 +18,7 @@ describe('module loader', () => { connections: [{ type: 'mysql' }, { type: 'postgresql' }], storage: { type: 'postgresql' } }; - const modules = await loadModules(config); + const modules = await loadModules(config as any); expect(modules.length).toBe(3); expect(modules[0]).toBeInstanceOf(MySQLModule); @@ -32,7 +32,7 @@ describe('module loader', () => { storage: { type: 'postgresql' } }; - const modules = await loadModules(config); + const modules = await loadModules(config as any); // Expect 3 modules: mysql, postgresql, postgresql-storage expect(modules.length).toBe(3); @@ -47,27 +47,28 @@ describe('module loader', () => { storage: { type: 'postgresql' } }; - await expect(loadModules(config)).rejects.toThrowError(); + await expect(loadModules(config as any)).rejects.toThrowError(); }); it('should throw an error if one dynamic connection module import fails', async () => { - vi.doMock('../../../src/util/module-loader.js', async (importOriginal) => { - const mod = await importOriginal(); - mod.ConnectionModuleMap.mysql = () => - import('@powersync/service-module-mysql').then(() => { - throw new Error('Failed to load MySQL module'); - }); - return mod; + vi.doMock('@powersync/service-module-mysql', async (importOriginal) => { + return { + MySQLModule: class { + constructor() { + throw new Error('Failed to load MySQL module!'); + } + } + }; }); - const { loadModules } = await import('../../../src/util/module-loader.js'); + const { loadModules } = await import('../../../lib/util/module-loader.js'); const config: MockConfig = { connections: [{ type: 'mysql' }], storage: { type: 'mongodb' } }; - await expect(loadModules(config)).rejects.toThrowError('Failed to load MySQL module'); + await expect(loadModules(config as any)).rejects.toThrowError('Failed to load MySQL module'); vi.doUnmock('@powersync/service-module-mysql'); }); From b4b19232e6ca6c62056493c01392b0741bd0cde7 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 4 Dec 2025 12:00:02 +0200 Subject: [PATCH 6/7] Move core module loading logic to service-core. --- packages/service-core/src/modules/loader.ts | 47 ++++++++ .../service-core/src/modules/modules-index.ts | 1 + .../test/src/module-loader.test.ts | 102 ++++++++++++++++++ service/package.json | 5 +- service/src/runners/server.ts | 4 +- service/src/runners/stream-worker.ts | 4 +- service/src/runners/unified-runner.ts | 4 +- service/src/util/module-loader.ts | 54 ---------- service/src/util/modules.ts | 15 +++ service/test/src/util/module-loader.test.ts | 75 ------------- 10 files changed, 173 insertions(+), 138 deletions(-) create mode 100644 packages/service-core/src/modules/loader.ts create mode 100644 packages/service-core/test/src/module-loader.test.ts delete mode 100644 service/src/util/module-loader.ts create mode 100644 service/src/util/modules.ts delete mode 100644 service/test/src/util/module-loader.test.ts diff --git a/packages/service-core/src/modules/loader.ts b/packages/service-core/src/modules/loader.ts new file mode 100644 index 000000000..db7799053 --- /dev/null +++ b/packages/service-core/src/modules/loader.ts @@ -0,0 +1,47 @@ +import { ResolvedPowerSyncConfig } from '../util/util-index.js'; +import { AbstractModule } from './AbstractModule.js'; + +interface DynamicModuleMap { + [key: string]: () => Promise; +} + +export interface ModuleLoaders { + storage: DynamicModuleMap; + connection: DynamicModuleMap; +} +/** + * Utility function to dynamically load and instantiate modules. + */ +export async function loadModules(config: ResolvedPowerSyncConfig, loaders: ModuleLoaders) { + const requiredConnections = [...new Set(config.connections?.map((connection) => connection.type) || [])]; + const missingConnectionModules: string[] = []; + const modulePromises: Promise[] = []; + + // 1. Map connection types to their module loading promises making note of any + // missing connection types. + requiredConnections.forEach((connectionType) => { + const modulePromise = loaders.connection[connectionType]; + if (modulePromise !== undefined) { + modulePromises.push(modulePromise()); + } else { + missingConnectionModules.push(connectionType); + } + }); + + // Fail if any connection types are not found. + if (missingConnectionModules.length > 0) { + throw new Error(`Invalid connection types: "${[...missingConnectionModules].join(', ')}"`); + } + + if (loaders.storage[config.storage.type] !== undefined) { + modulePromises.push(loaders.storage[config.storage.type]()); + } else { + throw new Error(`Invalid storage type: "${config.storage.type}"`); + } + + // 2. Dynamically import and instantiate module classes and resolve all promises + // raising errors if any modules could not be imported. + const moduleInstances = await Promise.all(modulePromises); + + return moduleInstances; +} diff --git a/packages/service-core/src/modules/modules-index.ts b/packages/service-core/src/modules/modules-index.ts index 1b32990e8..2bea3bf4e 100644 --- a/packages/service-core/src/modules/modules-index.ts +++ b/packages/service-core/src/modules/modules-index.ts @@ -1,2 +1,3 @@ export * from './ModuleManager.js'; export * from './AbstractModule.js'; +export * from './loader.js'; diff --git a/packages/service-core/test/src/module-loader.test.ts b/packages/service-core/test/src/module-loader.test.ts new file mode 100644 index 000000000..102086889 --- /dev/null +++ b/packages/service-core/test/src/module-loader.test.ts @@ -0,0 +1,102 @@ +import { AbstractModule, loadModules, ServiceContextContainer, TearDownOptions } from '@/index.js'; +import { describe, expect, it, vi } from 'vitest'; + +interface MockConfig { + connections?: { type: string }[]; + storage: { type: string }; +} + +class MockMySQLModule extends AbstractModule { + constructor() { + super({ name: 'MySQLModule' }); + } + async initialize(context: ServiceContextContainer): Promise {} + async teardown(options: TearDownOptions): Promise {} +} +class MockPostgresModule extends AbstractModule { + constructor() { + super({ name: 'PostgresModule' }); + } + async initialize(context: ServiceContextContainer): Promise {} + async teardown(options: TearDownOptions): Promise {} +} +class MockPostgresStorageModule extends AbstractModule { + constructor() { + super({ name: 'PostgresStorageModule' }); + } + async initialize(context: ServiceContextContainer): Promise {} + async teardown(options: TearDownOptions): Promise {} +} +const mockLoaders = { + connection: { + mysql: async () => { + return new MockMySQLModule(); + }, + postgresql: async () => { + return new MockPostgresModule(); + } + }, + storage: { + postgresql: async () => { + return new MockPostgresStorageModule(); + } + } +}; + +describe('module loader', () => { + it('should load all modules defined in connections and storage', async () => { + const config: MockConfig = { + connections: [{ type: 'mysql' }, { type: 'postgresql' }], + storage: { type: 'postgresql' } + }; + + const modules = await loadModules(config as any, mockLoaders); + + expect(modules.length).toBe(3); + expect(modules[0]).toBeInstanceOf(MockMySQLModule); + expect(modules[1]).toBeInstanceOf(MockPostgresModule); + expect(modules[2]).toBeInstanceOf(MockPostgresStorageModule); + }); + + it('should handle duplicate connection types (e.g., mysql used twice)', async () => { + const config: MockConfig = { + connections: [{ type: 'mysql' }, { type: 'postgresql' }, { type: 'mysql' }], // mysql duplicated + storage: { type: 'postgresql' } + }; + + const modules = await loadModules(config as any, mockLoaders); + + // Expect 3 modules: mysql, postgresql, postgresql-storage + expect(modules.length).toBe(3); + expect(modules.filter((m) => m instanceof MockMySQLModule).length).toBe(1); + expect(modules.filter((m) => m instanceof MockPostgresModule).length).toBe(1); + expect(modules.filter((m) => m instanceof MockPostgresStorageModule).length).toBe(1); + }); + + it('should throw an error if any modules are not found in ModuleMap', async () => { + const config: MockConfig = { + connections: [{ type: 'mysql' }, { type: 'redis' }], + storage: { type: 'postgresql' } + }; + + await expect(loadModules(config as any, mockLoaders)).rejects.toThrowError(); + }); + + it('should throw an error if one dynamic connection module import fails', async () => { + const config: MockConfig = { + connections: [{ type: 'mysql' }], + storage: { type: 'postgresql' } + }; + + const loaders = { + connection: { + mysql: async () => { + throw new Error('Failed to load MySQL module'); + } + }, + storage: mockLoaders.storage + }; + + await expect(loadModules(config as any, loaders)).rejects.toThrowError('Failed to load MySQL module'); + }); +}); diff --git a/service/package.json b/service/package.json index acec4eebf..5c63539e9 100644 --- a/service/package.json +++ b/service/package.json @@ -7,8 +7,7 @@ "scripts": { "build": "tsc -b", "watch": "nodemon -w ../ -e ts -e js --delay 1 -x node --loader ts-node/esm src/entry.ts start", - "clean": "rm -rf ./lib && tsc -b --clean", - "test": "vitest" + "clean": "rm -rf ./lib && tsc -b --clean" }, "dependencies": { "@powersync/service-core": "workspace:*", @@ -29,4 +28,4 @@ "npm-check-updates": "^16.14.4", "ts-node": "^10.9.1" } -} +} \ No newline at end of file diff --git a/service/src/runners/server.ts b/service/src/runners/server.ts index 92c092fc4..6484cfaab 100644 --- a/service/src/runners/server.ts +++ b/service/src/runners/server.ts @@ -1,8 +1,8 @@ import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; -import { loadModules } from '../util/module-loader.js'; import { logBooting } from '../util/version.js'; +import { DYNAMIC_MODULES } from '../util/modules.js'; /** * Starts an API server @@ -13,7 +13,7 @@ export async function startServer(runnerConfig: core.utils.RunnerConfig) { const config = await core.utils.loadConfig(runnerConfig); const moduleManager = container.getImplementation(core.modules.ModuleManager); - const modules = await loadModules(config); + const modules = await core.loadModules(config, DYNAMIC_MODULES); if (modules.length > 0) { moduleManager.register(modules); } diff --git a/service/src/runners/stream-worker.ts b/service/src/runners/stream-worker.ts index 9134fa549..8d4ebaf74 100644 --- a/service/src/runners/stream-worker.ts +++ b/service/src/runners/stream-worker.ts @@ -1,8 +1,8 @@ import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; -import { loadModules } from '../util/module-loader.js'; import { logBooting } from '../util/version.js'; +import { DYNAMIC_MODULES } from '../util/modules.js'; /** * Configures the replication portion on a {@link serviceContext} @@ -24,7 +24,7 @@ export const startStreamRunner = async (runnerConfig: core.utils.RunnerConfig) = const config = await core.utils.loadConfig(runnerConfig); const moduleManager = container.getImplementation(core.modules.ModuleManager); - const modules = await loadModules(config); + const modules = await core.loadModules(config, DYNAMIC_MODULES); if (modules.length > 0) { moduleManager.register(modules); } diff --git a/service/src/runners/unified-runner.ts b/service/src/runners/unified-runner.ts index 648aaf98f..88b93c1f5 100644 --- a/service/src/runners/unified-runner.ts +++ b/service/src/runners/unified-runner.ts @@ -1,9 +1,9 @@ import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; -import { loadModules } from '../util/module-loader.js'; import { logBooting } from '../util/version.js'; import { registerReplicationServices } from './stream-worker.js'; +import { DYNAMIC_MODULES } from '../util/modules.js'; /** * Starts an API server @@ -14,7 +14,7 @@ export const startUnifiedRunner = async (runnerConfig: core.utils.RunnerConfig) const config = await core.utils.loadConfig(runnerConfig); const moduleManager = container.getImplementation(core.modules.ModuleManager); - const modules = await loadModules(config); + const modules = await core.loadModules(config, DYNAMIC_MODULES); if (modules.length > 0) { moduleManager.register(modules); } diff --git a/service/src/util/module-loader.ts b/service/src/util/module-loader.ts deleted file mode 100644 index 6b245b7ce..000000000 --- a/service/src/util/module-loader.ts +++ /dev/null @@ -1,54 +0,0 @@ -import * as core from '@powersync/service-core'; - -interface DynamicModuleMap { - [key: string]: () => Promise; -} - -export const CONNECTION_MODULE_MAP: DynamicModuleMap = { - mongodb: () => import('@powersync/service-module-mongodb').then((module) => new module.MongoModule()), - mysql: () => import('@powersync/service-module-mysql').then((module) => new module.MySQLModule()), - postgresql: () => import('@powersync/service-module-postgres').then((module) => new module.PostgresModule()) -}; - -const STORAGE_MODULE_MAP: DynamicModuleMap = { - mongodb: () => import('@powersync/service-module-mongodb-storage').then((module) => new module.MongoStorageModule()), - postgresql: () => - import('@powersync/service-module-postgres-storage').then((module) => new module.PostgresStorageModule()) -}; - -/** - * Utility function to dynamically load and instantiate modules. - */ -export async function loadModules(config: core.ResolvedPowerSyncConfig) { - const requiredConnections = [...new Set(config.connections?.map((connection) => connection.type) || [])]; - const missingConnectionModules: string[] = []; - const modulePromises: Promise[] = []; - - // 1. Map connection types to their module loading promises making note of any - // missing connection types. - requiredConnections.forEach((connectionType) => { - const modulePromise = CONNECTION_MODULE_MAP[connectionType]; - if (modulePromise !== undefined) { - modulePromises.push(modulePromise()); - } else { - missingConnectionModules.push(connectionType); - } - }); - - // Fail if any connection types are not found. - if (missingConnectionModules.length > 0) { - throw new Error(`Invalid connection types: "${[...missingConnectionModules].join(', ')}"`); - } - - if (STORAGE_MODULE_MAP[config.storage.type] !== undefined) { - modulePromises.push(STORAGE_MODULE_MAP[config.storage.type]()); - } else { - throw new Error(`Invalid storage type: "${config.storage.type}"`); - } - - // 2. Dynamically import and instantiate module classes and resolve all promises - // raising errors if any modules could not be imported. - const moduleInstances = await Promise.all(modulePromises); - - return moduleInstances; -} diff --git a/service/src/util/modules.ts b/service/src/util/modules.ts new file mode 100644 index 000000000..5ef36858c --- /dev/null +++ b/service/src/util/modules.ts @@ -0,0 +1,15 @@ +import * as core from '@powersync/service-core'; + +export const DYNAMIC_MODULES: core.ModuleLoaders = { + connection: { + mongodb: () => import('@powersync/service-module-mongodb').then((module) => new module.MongoModule()), + mysql: () => import('@powersync/service-module-mysql').then((module) => new module.MySQLModule()), + postgresql: () => import('@powersync/service-module-postgres').then((module) => new module.PostgresModule()) + }, + storage: { + mongodb: () => + import('@powersync/service-module-mongodb-storage').then((module) => new module.MongoStorageModule()), + postgresql: () => + import('@powersync/service-module-postgres-storage').then((module) => new module.PostgresStorageModule()) + } +}; diff --git a/service/test/src/util/module-loader.test.ts b/service/test/src/util/module-loader.test.ts deleted file mode 100644 index bdb2d6adc..000000000 --- a/service/test/src/util/module-loader.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { vi, describe, test, expect, it } from 'vitest'; - -import { logger } from '@powersync/lib-services-framework'; - -import { MySQLModule } from '@powersync/service-module-mysql'; -import { PostgresModule } from '@powersync/service-module-postgres'; -import { PostgresStorageModule } from '@powersync/service-module-postgres-storage'; -import { loadModules } from '../../../src/util/module-loader.js'; - -interface MockConfig { - connections?: { type: string }[]; - storage: { type: string }; -} - -describe('module loader', () => { - it('should load all modules defined in connections and storage', async () => { - const config: MockConfig = { - connections: [{ type: 'mysql' }, { type: 'postgresql' }], - storage: { type: 'postgresql' } - }; - const modules = await loadModules(config as any); - - expect(modules.length).toBe(3); - expect(modules[0]).toBeInstanceOf(MySQLModule); - expect(modules[1]).toBeInstanceOf(PostgresModule); - expect(modules[2]).toBeInstanceOf(PostgresStorageModule); - }); - - it('should handle duplicate connection types (e.g., mysql used twice)', async () => { - const config: MockConfig = { - connections: [{ type: 'mysql' }, { type: 'postgresql' }, { type: 'mysql' }], // mysql duplicated - storage: { type: 'postgresql' } - }; - - const modules = await loadModules(config as any); - - // Expect 3 modules: mysql, postgresql, postgresql-storage - expect(modules.length).toBe(3); - expect(modules.filter((m) => m instanceof MySQLModule).length).toBe(1); - expect(modules.filter((m) => m instanceof PostgresModule).length).toBe(1); - expect(modules.filter((m) => m instanceof PostgresStorageModule).length).toBe(1); - }); - - it('should throw an error if any modules are not found in ModuleMap', async () => { - const config: MockConfig = { - connections: [{ type: 'mysql' }, { type: 'redis' }], - storage: { type: 'postgresql' } - }; - - await expect(loadModules(config as any)).rejects.toThrowError(); - }); - - it('should throw an error if one dynamic connection module import fails', async () => { - vi.doMock('@powersync/service-module-mysql', async (importOriginal) => { - return { - MySQLModule: class { - constructor() { - throw new Error('Failed to load MySQL module!'); - } - } - }; - }); - - const { loadModules } = await import('../../../lib/util/module-loader.js'); - - const config: MockConfig = { - connections: [{ type: 'mysql' }], - storage: { type: 'mongodb' } - }; - - await expect(loadModules(config as any)).rejects.toThrowError('Failed to load MySQL module'); - - vi.doUnmock('@powersync/service-module-mysql'); - }); -}); From 1d7d913f93cbc8b26db3209aaedff012d9e40889 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 4 Dec 2025 12:06:07 +0200 Subject: [PATCH 7/7] Skip docker login on PRs from forks. --- .github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c2a8e3604..50a7b931d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,6 +19,7 @@ jobs: uses: actions/checkout@v5 - name: Login to Docker Hub + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -48,6 +49,7 @@ jobs: - uses: actions/checkout@v5 - name: Login to Docker Hub + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -95,6 +97,7 @@ jobs: - uses: actions/checkout@v5 - name: Login to Docker Hub + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -171,6 +174,7 @@ jobs: - uses: actions/checkout@v5 - name: Login to Docker Hub + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -244,6 +248,7 @@ jobs: - uses: actions/checkout@v4 - name: Login to Docker Hub + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }}