Dynamically load connection modules for reduced memory usage

.changeset/perfect-plants-report.md

@@ -0,0 +1,5 @@
+---
+'@powersync/service-image': minor
+---
+
+Dynamically load connection modules for reduced memory usage

service/package.json

@@ -7,7 +7,8 @@
   "scripts": {
     "build": "tsc -b",
     "watch": "nodemon -w ../ -e ts -e js --delay 1 -x node --loader ts-node/esm src/entry.ts start",
-    "clean": "rm -rf ./lib && tsc -b --clean"
+    "clean": "rm -rf ./lib && tsc -b --clean",
+    "test": "vitest"
   },
   "dependencies": {
     "@powersync/service-core": "workspace:*",

service/src/entry.ts

@@ -17,14 +17,7 @@ container.registerDefaults();
 container.register(ContainerImplementation.REPORTER, createSentryReporter());
 
 const moduleManager = new core.modules.ModuleManager();
-moduleManager.register([
-  new CoreModule(),
-  new MongoModule(),
-  new MongoStorageModule(),
-  new MySQLModule(),
-  new PostgresModule(),
-  new PostgresStorageModule()
-]);
+moduleManager.register([new CoreModule(), new MongoModule(), new MongoStorageModule()]);
 // This is a bit of a hack. Commands such as the teardown command or even migrations might
 // want access to the ModuleManager in order to use modules
 container.register(core.ModuleManager, moduleManager);

service/src/runners/server.ts

@@ -1,5 +1,7 @@
 import { container, logger } from '@powersync/lib-services-framework';
 import * as core from '@powersync/service-core';
+
+import { loadModules } from '../util/module-loader.js';
 import { logBooting } from '../util/version.js';
 
 /**
@@ -9,12 +11,18 @@ export async function startServer(runnerConfig: core.utils.RunnerConfig) {
   logBooting('API Container');
 
   const config = await core.utils.loadConfig(runnerConfig);
+
+  const moduleManager = container.getImplementation(core.modules.ModuleManager);
+  const modules = await loadModules(config);
+  if (modules.length > 0) {
+    moduleManager.register(modules);
+  }
+
   const serviceContext = new core.system.ServiceContextContainer({
     serviceMode: core.system.ServiceContextMode.API,
     configuration: config
   });
 
-  const moduleManager = container.getImplementation(core.modules.ModuleManager);
   await moduleManager.initialize(serviceContext);
 
   logger.info('Starting service...');

service/src/runners/stream-worker.ts

@@ -1,5 +1,7 @@
 import { container, logger } from '@powersync/lib-services-framework';
 import * as core from '@powersync/service-core';
+
+import { loadModules } from '../util/module-loader.js';
 import { logBooting } from '../util/version.js';
 
 /**
@@ -20,15 +22,20 @@ export const startStreamRunner = async (runnerConfig: core.utils.RunnerConfig) =
   logBooting('Replication Container');
 
   const config = await core.utils.loadConfig(runnerConfig);
+
+  const moduleManager = container.getImplementation(core.modules.ModuleManager);
+  const modules = await loadModules(config);
+  if (modules.length > 0) {
+    moduleManager.register(modules);
+  }
+
   // Self-hosted version allows for automatic migrations
   const serviceContext = new core.system.ServiceContextContainer({
     serviceMode: core.system.ServiceContextMode.SYNC,
     configuration: config
   });
-
   registerReplicationServices(serviceContext);
 
-  const moduleManager = container.getImplementation(core.modules.ModuleManager);
   await moduleManager.initialize(serviceContext);
 
   // Ensure automatic migrations

service/src/runners/unified-runner.ts

@@ -1,6 +1,7 @@
 import { container, logger } from '@powersync/lib-services-framework';
 import * as core from '@powersync/service-core';
 
+import { loadModules } from '../util/module-loader.js';
 import { logBooting } from '../util/version.js';
 import { registerReplicationServices } from './stream-worker.js';
 
@@ -11,14 +12,19 @@ export const startUnifiedRunner = async (runnerConfig: core.utils.RunnerConfig)
   logBooting('Unified Container');
 
   const config = await core.utils.loadConfig(runnerConfig);
+
+  const moduleManager = container.getImplementation(core.modules.ModuleManager);
+  const modules = await loadModules(config);
+  if (modules.length > 0) {
+    moduleManager.register(modules);
+  }
+
   const serviceContext = new core.system.ServiceContextContainer({
     serviceMode: core.system.ServiceContextMode.UNIFIED,
     configuration: config
   });
-
   registerReplicationServices(serviceContext);
 
-  const moduleManager = container.getImplementation(core.modules.ModuleManager);
   await moduleManager.initialize(serviceContext);
 
   await core.migrations.ensureAutomaticMigrations({

service/src/util/module-loader.ts

@@ -0,0 +1,56 @@
+import { container, logger } from '@powersync/lib-services-framework';
+import * as core from '@powersync/service-core';
+
+interface DynamicModuleMap {
+  [key: string]: () => Promise<any>;
+}
+
+const ModuleMap: DynamicModuleMap = {
+  mysql: () => import('@powersync/service-module-mysql').then((module) => module.MySQLModule),
+  postgresql: () => import('@powersync/service-module-postgres').then((module) => module.PostgresModule),
+  'postgresql-storage': () =>
+    import('@powersync/service-module-postgres-storage').then((module) => module.PostgresStorageModule)
+};
+
+/**
+ * Utility function to dynamically load and instantiate modules.
+ * This function can optionally be moved to its own file (e.g., module-loader.ts)
+ * for better separation of concerns if the file gets much larger.
+ */
+export async function loadModules(config: core.ResolvedPowerSyncConfig) {
+  // 1. Determine required connections: Unique types from connections + storage type
+  const requiredConnections = [
+    ...new Set(config.connections?.map((connection) => connection.type) || []),
+    `${config.storage.type}-storage` // Using template literal is clear
+  ];
+
+  // 2. Map connection types to their module loading promises
+  const modulePromises = requiredConnections.map(async (connectionType) => {
+    // Exclude 'mongo' connections explicitly early
+    if (connectionType.startsWith('mongo')) {
+      return null; // Return null for filtering later
+    }
+
+    const modulePromise = ModuleMap[connectionType];
+
+    // Check if a module is defined for the connection type
+    if (!modulePromise) {
+      logger.warn(`No module defined in ModuleMap for connection type: ${connectionType}`);
+      return null;
+    }
+
+    try {
+      // Dynamically import and instantiate the class
+      const ModuleClass = await modulePromise();
+      return new ModuleClass();
+    } catch (error) {
+      // Log an error if the dynamic import fails (e.g., module not installed)
+      logger.error(`Failed to load module for ${connectionType}:`, error);
+      return null;
+    }
+  });
+
+  // 3. Resolve all promises and filter out nulls/undefineds
+  const moduleInstances = await Promise.all(modulePromises);
+  return moduleInstances.filter((instance) => instance !== null); // Filter out nulls from excluded or failed imports
+}

service/test/src/util/module-loader.test.ts

@@ -0,0 +1,85 @@
+import { describe, test, expect, it } from 'vitest';
+
+import { logger } from '@powersync/lib-services-framework';
+
+import { MySQLModule } from '@powersync/service-module-mysql';
+import { PostgresModule } from '@powersync/service-module-postgres';
+import { PostgresStorageModule } from '@powersync/service-module-postgres-storage';
+import { loadModules } from '../../../src/util/module-loader.js';
+
+interface MockConfig {
+  connections?: MockConnection[];
+  storage: { type: string };
+}
+
+describe('module loader', () => {
+  it('should load all modules defined in connections and storage', async () => {
+    const config: MockConfig = {
+      connections: [{ type: 'mysql' }, { type: 'postgresql' }],
+      storage: { type: 'postgresql' } // This should result in 'postgresql-storage'
+    };
+
+    const modules = await loadModules(config);
+
+    expect(modules.length).toBe(3);
+    expect(modules[0]).toBeInstanceOf(MySQLModule);
+    expect(modules[1]).toBeInstanceOf(PostgresModule);
+    expect(modules[2]).toBeInstanceOf(PostgresStorageModule);
+  });
+
+  it('should handle duplicate connection types (e.g., mysql used twice)', async () => {
+    const config: MockConfig = {
+      connections: [{ type: 'mysql' }, { type: 'postgresql' }, { type: 'mysql' }], // mysql duplicated
+      storage: { type: 'postgresql' }
+    };
+
+    const modules = await loadModules(config);
+
+    // Expect 3 modules: mysql, postgresql, postgresql-storage
+    expect(modules.length).toBe(3);
+    expect(modules.filter((m) => m instanceof MySQLModule).length).toBe(1);
+    expect(modules.filter((m) => m instanceof PostgresModule).length).toBe(1);
+    expect(modules.filter((m) => m instanceof PostgresStorageModule).length).toBe(1);
+  });
+
+  it('should exclude connections starting with "mongo"', async () => {
+    const config: MockConfig = {
+      connections: [{ type: 'mysql' }, { type: 'mongodb' }], // mongodb should be ignored
+      storage: { type: 'postgresql' }
+    };
+
+    const modules = await loadModules(config);
+
+    // Expect 2 modules: mysql and postgresql-storage
+    expect(modules.length).toBe(2);
+    expect(modules[0]).toBeInstanceOf(MySQLModule);
+    expect(modules[1]).toBeInstanceOf(PostgresStorageModule);
+    expect(modules.filter((m) => m instanceof PostgresModule).length).toBe(0);
+  });
+
+  it('should filter out modules not found in ModuleMap', async () => {
+    const config: MockConfig = {
+      connections: [{ type: 'mysql' }, { type: 'redis' }], // unknown-db is missing
+      storage: { type: 'postgresql' }
+    };
+
+    const modules = await loadModules(config);
+
+    // Expect 2 modules: mysql and postgresql-storage
+    expect(modules.length).toBe(2);
+    expect(modules.every((m) => m instanceof MySQLModule || m instanceof PostgresStorageModule)).toBe(true);
+  });
+
+  it('should filter out modules that fail to import and log an error', async () => {
+    const config: MockConfig = {
+      connections: [{ type: 'mysql' }, { type: 'failing-module' }], // failing-module rejects promise
+      storage: { type: 'postgresql' }
+    };
+
+    const modules = await loadModules(config);
+
+    // Expect 2 modules: mysql and postgresql-storage
+    expect(modules.length).toBe(2);
+    expect(modules.filter((m) => m instanceof MySQLModule).length).toBe(1);
+  });
+});