Dynamically load connection modules for reduced memory usage

.changeset/perfect-plants-report.md

@@ -0,0 +1,5 @@
+---
+'@powersync/service-image': minor
+---
+
+Dynamically load connection modules for reduced memory usage

service/package.json

@@ -7,7 +7,8 @@
   "scripts": {
     "build": "tsc -b",
     "watch": "nodemon -w ../ -e ts -e js --delay 1 -x node --loader ts-node/esm src/entry.ts start",
-    "clean": "rm -rf ./lib && tsc -b --clean"
+    "clean": "rm -rf ./lib && tsc -b --clean",
+    "test": "vitest"
   },
   "dependencies": {
     "@powersync/service-core": "workspace:*",

service/src/entry.ts

@@ -2,11 +2,6 @@ import { container, ContainerImplementation } from '@powersync/lib-services-fram
 import * as core from '@powersync/service-core';
 
 import { CoreModule } from '@powersync/service-module-core';
-import { MongoModule } from '@powersync/service-module-mongodb';
-import { MongoStorageModule } from '@powersync/service-module-mongodb-storage';
-import { MySQLModule } from '@powersync/service-module-mysql';
-import { PostgresModule } from '@powersync/service-module-postgres';
-import { PostgresStorageModule } from '@powersync/service-module-postgres-storage';
 import { startServer } from './runners/server.js';
 import { startStreamRunner } from './runners/stream-worker.js';
 import { startUnifiedRunner } from './runners/unified-runner.js';
@@ -17,14 +12,7 @@ container.registerDefaults();
 container.register(ContainerImplementation.REPORTER, createSentryReporter());
 
 const moduleManager = new core.modules.ModuleManager();
-moduleManager.register([
-  new CoreModule(),
-  new MongoModule(),
-  new MongoStorageModule(),
-  new MySQLModule(),
-  new PostgresModule(),
-  new PostgresStorageModule()
-]);
+moduleManager.register([new CoreModule()]);
 // This is a bit of a hack. Commands such as the teardown command or even migrations might
 // want access to the ModuleManager in order to use modules
 container.register(core.ModuleManager, moduleManager);

service/src/runners/server.ts

@@ -1,5 +1,7 @@
 import { container, logger } from '@powersync/lib-services-framework';
 import * as core from '@powersync/service-core';
+
+import { loadModules } from '../util/module-loader.js';
 import { logBooting } from '../util/version.js';
 
 /**
@@ -9,12 +11,18 @@ export async function startServer(runnerConfig: core.utils.RunnerConfig) {
   logBooting('API Container');
 
   const config = await core.utils.loadConfig(runnerConfig);
+
+  const moduleManager = container.getImplementation(core.modules.ModuleManager);
+  const modules = await loadModules(config);
+  if (modules.length > 0) {
+    moduleManager.register(modules);
+  }
+
   const serviceContext = new core.system.ServiceContextContainer({
     serviceMode: core.system.ServiceContextMode.API,
     configuration: config
   });
 
-  const moduleManager = container.getImplementation(core.modules.ModuleManager);
   await moduleManager.initialize(serviceContext);
 
   logger.info('Starting service...');

service/src/runners/stream-worker.ts

@@ -1,5 +1,7 @@
 import { container, logger } from '@powersync/lib-services-framework';
 import * as core from '@powersync/service-core';
+
+import { loadModules } from '../util/module-loader.js';
 import { logBooting } from '../util/version.js';
 
 /**
@@ -20,15 +22,20 @@ export const startStreamRunner = async (runnerConfig: core.utils.RunnerConfig) =
   logBooting('Replication Container');
 
   const config = await core.utils.loadConfig(runnerConfig);
+
+  const moduleManager = container.getImplementation(core.modules.ModuleManager);
+  const modules = await loadModules(config);
+  if (modules.length > 0) {
+    moduleManager.register(modules);
+  }
+
   // Self-hosted version allows for automatic migrations
   const serviceContext = new core.system.ServiceContextContainer({
     serviceMode: core.system.ServiceContextMode.SYNC,
     configuration: config
   });
-
   registerReplicationServices(serviceContext);
 
-  const moduleManager = container.getImplementation(core.modules.ModuleManager);
   await moduleManager.initialize(serviceContext);
 
   // Ensure automatic migrations

service/src/runners/unified-runner.ts

@@ -1,6 +1,7 @@
 import { container, logger } from '@powersync/lib-services-framework';
 import * as core from '@powersync/service-core';
 
+import { loadModules } from '../util/module-loader.js';
 import { logBooting } from '../util/version.js';
 import { registerReplicationServices } from './stream-worker.js';
 
@@ -11,14 +12,19 @@ export const startUnifiedRunner = async (runnerConfig: core.utils.RunnerConfig)
   logBooting('Unified Container');
 
   const config = await core.utils.loadConfig(runnerConfig);
+
+  const moduleManager = container.getImplementation(core.modules.ModuleManager);
+  const modules = await loadModules(config);
+  if (modules.length > 0) {
+    moduleManager.register(modules);
+  }
+
   const serviceContext = new core.system.ServiceContextContainer({
     serviceMode: core.system.ServiceContextMode.UNIFIED,
     configuration: config
   });
-
   registerReplicationServices(serviceContext);
 
-  const moduleManager = container.getImplementation(core.modules.ModuleManager);
   await moduleManager.initialize(serviceContext);
 
   await core.migrations.ensureAutomaticMigrations({

service/src/util/module-loader.ts

@@ -0,0 +1,54 @@
+import * as core from '@powersync/service-core';
+
+interface DynamicModuleMap {
+  [key: string]: () => Promise<core.AbstractModule>;
+}
+
+export const CONNECTION_MODULE_MAP: DynamicModuleMap = {
+  mongodb: () => import('@powersync/service-module-mongodb').then((module) => new module.MongoModule()),
+  mysql: () => import('@powersync/service-module-mysql').then((module) => new module.MySQLModule()),
+  postgresql: () => import('@powersync/service-module-postgres').then((module) => new module.PostgresModule())
+};
+
+const STORAGE_MODULE_MAP: DynamicModuleMap = {
+  mongodb: () => import('@powersync/service-module-mongodb-storage').then((module) => new module.MongoStorageModule()),
+  postgresql: () =>
+    import('@powersync/service-module-postgres-storage').then((module) => new module.PostgresStorageModule())
+};
+
+/**
+ * Utility function to dynamically load and instantiate modules.
+ */
+export async function loadModules(config: core.ResolvedPowerSyncConfig) {
+  const requiredConnections = [...new Set(config.connections?.map((connection) => connection.type) || [])];
+  const missingConnectionModules: string[] = [];
+  const modulePromises: Promise<core.AbstractModule>[] = [];
+
+  // 1. Map connection types to their module loading promises making note of any
+  // missing connection types.
+  requiredConnections.forEach((connectionType) => {
+    const modulePromise = CONNECTION_MODULE_MAP[connectionType];
+    if (modulePromise !== undefined) {
+      modulePromises.push(modulePromise());
+    } else {
+      missingConnectionModules.push(connectionType);
+    }
+  });
+
+  // Fail if any connection types are not found.
+  if (missingConnectionModules.length > 0) {
+    throw new Error(`Invalid connection types: "${[...missingConnectionModules].join(', ')}"`);
+  }
+
+  if (STORAGE_MODULE_MAP[config.storage.type] !== undefined) {
+    modulePromises.push(STORAGE_MODULE_MAP[config.storage.type]());
+  } else {
+    throw new Error(`Invalid storage type: "${config.storage.type}"`);
+  }
+
+  // 2. Dynamically import and instantiate module classes and resolve all promises
+  // raising errors if any modules could not be imported.
+  const moduleInstances = await Promise.all(modulePromises);
+
+  return moduleInstances;
+}

service/test/src/util/module-loader.test.ts

@@ -0,0 +1,75 @@
+import { vi, describe, test, expect, it } from 'vitest';
+
+import { logger } from '@powersync/lib-services-framework';
+
+import { MySQLModule } from '@powersync/service-module-mysql';
+import { PostgresModule } from '@powersync/service-module-postgres';
+import { PostgresStorageModule } from '@powersync/service-module-postgres-storage';
+import { loadModules } from '../../../src/util/module-loader.js';
+
+interface MockConfig {
+  connections?: { type: string }[];
+  storage: { type: string };
+}
+
+describe('module loader', () => {
+  it('should load all modules defined in connections and storage', async () => {
+    const config: MockConfig = {
+      connections: [{ type: 'mysql' }, { type: 'postgresql' }],
+      storage: { type: 'postgresql' }
+    };
+    const modules = await loadModules(config as any);
+
+    expect(modules.length).toBe(3);
+    expect(modules[0]).toBeInstanceOf(MySQLModule);
+    expect(modules[1]).toBeInstanceOf(PostgresModule);
+    expect(modules[2]).toBeInstanceOf(PostgresStorageModule);
+  });
+
+  it('should handle duplicate connection types (e.g., mysql used twice)', async () => {
+    const config: MockConfig = {
+      connections: [{ type: 'mysql' }, { type: 'postgresql' }, { type: 'mysql' }], // mysql duplicated
+      storage: { type: 'postgresql' }
+    };
+
+    const modules = await loadModules(config as any);
+
+    // Expect 3 modules: mysql, postgresql, postgresql-storage
+    expect(modules.length).toBe(3);
+    expect(modules.filter((m) => m instanceof MySQLModule).length).toBe(1);
+    expect(modules.filter((m) => m instanceof PostgresModule).length).toBe(1);
+    expect(modules.filter((m) => m instanceof PostgresStorageModule).length).toBe(1);
+  });
+
+  it('should throw an error if any modules are not found in ModuleMap', async () => {
+    const config: MockConfig = {
+      connections: [{ type: 'mysql' }, { type: 'redis' }],
+      storage: { type: 'postgresql' }
+    };
+
+    await expect(loadModules(config as any)).rejects.toThrowError();
+  });
+
+  it('should throw an error if one dynamic connection module import fails', async () => {
+    vi.doMock('@powersync/service-module-mysql', async (importOriginal) => {
+      return {
+        MySQLModule: class {
+          constructor() {
+            throw new Error('Failed to load MySQL module!');
+          }
+        }
+      };
+    });
+
+    const { loadModules } = await import('../../../lib/util/module-loader.js');
+
+    const config: MockConfig = {
+      connections: [{ type: 'mysql' }],
+      storage: { type: 'mongodb' }
+    };
+
+    await expect(loadModules(config as any)).rejects.toThrowError('Failed to load MySQL module');
+
+    vi.doUnmock('@powersync/service-module-mysql');
+  });
+});