Dynamically load connection modules for reduced memory usage

.changeset/perfect-plants-report.md

@@ -0,0 +1,5 @@
+---
+'@powersync/service-image': minor
+---
+
+Dynamically load connection modules for reduced memory usage

.github/workflows/test.yml

@@ -19,6 +19,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -48,6 +49,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -95,6 +97,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -171,6 +174,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -244,6 +248,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}

packages/service-core/src/modules/loader.ts

@@ -0,0 +1,47 @@
+import { ResolvedPowerSyncConfig } from '../util/util-index.js';
+import { AbstractModule } from './AbstractModule.js';
+
+interface DynamicModuleMap {
+  [key: string]: () => Promise<AbstractModule>;
+}
+
+export interface ModuleLoaders {
+  storage: DynamicModuleMap;
+  connection: DynamicModuleMap;
+}
+/**
+ * Utility function to dynamically load and instantiate modules.
+ */
+export async function loadModules(config: ResolvedPowerSyncConfig, loaders: ModuleLoaders) {
+  const requiredConnections = [...new Set(config.connections?.map((connection) => connection.type) || [])];
+  const missingConnectionModules: string[] = [];
+  const modulePromises: Promise<AbstractModule>[] = [];
+
+  // 1. Map connection types to their module loading promises making note of any
+  // missing connection types.
+  requiredConnections.forEach((connectionType) => {
+    const modulePromise = loaders.connection[connectionType];
+    if (modulePromise !== undefined) {
+      modulePromises.push(modulePromise());
+    } else {
+      missingConnectionModules.push(connectionType);
+    }
+  });
+
+  // Fail if any connection types are not found.
+  if (missingConnectionModules.length > 0) {
+    throw new Error(`Invalid connection types: "${[...missingConnectionModules].join(', ')}"`);
+  }
+
+  if (loaders.storage[config.storage.type] !== undefined) {
+    modulePromises.push(loaders.storage[config.storage.type]());
+  } else {
+    throw new Error(`Invalid storage type: "${config.storage.type}"`);
+  }
+
+  // 2. Dynamically import and instantiate module classes and resolve all promises
+  // raising errors if any modules could not be imported.
+  const moduleInstances = await Promise.all(modulePromises);
+
+  return moduleInstances;
+}

packages/service-core/src/modules/modules-index.ts

@@ -1,2 +1,3 @@
 export * from './ModuleManager.js';
 export * from './AbstractModule.js';
+export * from './loader.js';

packages/service-core/test/src/module-loader.test.ts

@@ -0,0 +1,102 @@
+import { AbstractModule, loadModules, ServiceContextContainer, TearDownOptions } from '@/index.js';
+import { describe, expect, it, vi } from 'vitest';
+
+interface MockConfig {
+  connections?: { type: string }[];
+  storage: { type: string };
+}
+
+class MockMySQLModule extends AbstractModule {
+  constructor() {
+    super({ name: 'MySQLModule' });
+  }
+  async initialize(context: ServiceContextContainer): Promise<void> {}
+  async teardown(options: TearDownOptions): Promise<void> {}
+}
+class MockPostgresModule extends AbstractModule {
+  constructor() {
+    super({ name: 'PostgresModule' });
+  }
+  async initialize(context: ServiceContextContainer): Promise<void> {}
+  async teardown(options: TearDownOptions): Promise<void> {}
+}
+class MockPostgresStorageModule extends AbstractModule {
+  constructor() {
+    super({ name: 'PostgresStorageModule' });
+  }
+  async initialize(context: ServiceContextContainer): Promise<void> {}
+  async teardown(options: TearDownOptions): Promise<void> {}
+}
+const mockLoaders = {
+  connection: {
+    mysql: async () => {
+      return new MockMySQLModule();
+    },
+    postgresql: async () => {
+      return new MockPostgresModule();
+    }
+  },
+  storage: {
+    postgresql: async () => {
+      return new MockPostgresStorageModule();
+    }
+  }
+};
+
+describe('module loader', () => {
+  it('should load all modules defined in connections and storage', async () => {
+    const config: MockConfig = {
+      connections: [{ type: 'mysql' }, { type: 'postgresql' }],
+      storage: { type: 'postgresql' }
+    };
+
+    const modules = await loadModules(config as any, mockLoaders);
+
+    expect(modules.length).toBe(3);
+    expect(modules[0]).toBeInstanceOf(MockMySQLModule);
+    expect(modules[1]).toBeInstanceOf(MockPostgresModule);
+    expect(modules[2]).toBeInstanceOf(MockPostgresStorageModule);
+  });
+
+  it('should handle duplicate connection types (e.g., mysql used twice)', async () => {
+    const config: MockConfig = {
+      connections: [{ type: 'mysql' }, { type: 'postgresql' }, { type: 'mysql' }], // mysql duplicated
+      storage: { type: 'postgresql' }
+    };
+
+    const modules = await loadModules(config as any, mockLoaders);
+
+    // Expect 3 modules: mysql, postgresql, postgresql-storage
+    expect(modules.length).toBe(3);
+    expect(modules.filter((m) => m instanceof MockMySQLModule).length).toBe(1);
+    expect(modules.filter((m) => m instanceof MockPostgresModule).length).toBe(1);
+    expect(modules.filter((m) => m instanceof MockPostgresStorageModule).length).toBe(1);
+  });
+
+  it('should throw an error if any modules are not found in ModuleMap', async () => {
+    const config: MockConfig = {
+      connections: [{ type: 'mysql' }, { type: 'redis' }],
+      storage: { type: 'postgresql' }
+    };
+
+    await expect(loadModules(config as any, mockLoaders)).rejects.toThrowError();
+  });
+
+  it('should throw an error if one dynamic connection module import fails', async () => {
+    const config: MockConfig = {
+      connections: [{ type: 'mysql' }],
+      storage: { type: 'postgresql' }
+    };
+
+    const loaders = {
+      connection: {
+        mysql: async () => {
+          throw new Error('Failed to load MySQL module');
+        }
+      },
+      storage: mockLoaders.storage
+    };
+
+    await expect(loadModules(config as any, loaders)).rejects.toThrowError('Failed to load MySQL module');
+  });
+});

service/package.json

@@ -28,4 +28,4 @@
     "npm-check-updates": "^16.14.4",
     "ts-node": "^10.9.1"
   }
-}
+}

service/src/entry.ts

@@ -2,11 +2,6 @@ import { container, ContainerImplementation } from '@powersync/lib-services-fram
 import * as core from '@powersync/service-core';
 
 import { CoreModule } from '@powersync/service-module-core';
-import { MongoModule } from '@powersync/service-module-mongodb';
-import { MongoStorageModule } from '@powersync/service-module-mongodb-storage';
-import { MySQLModule } from '@powersync/service-module-mysql';
-import { PostgresModule } from '@powersync/service-module-postgres';
-import { PostgresStorageModule } from '@powersync/service-module-postgres-storage';
 import { startServer } from './runners/server.js';
 import { startStreamRunner } from './runners/stream-worker.js';
 import { startUnifiedRunner } from './runners/unified-runner.js';
@@ -17,14 +12,7 @@ container.registerDefaults();
 container.register(ContainerImplementation.REPORTER, createSentryReporter());
 
 const moduleManager = new core.modules.ModuleManager();
-moduleManager.register([
-  new CoreModule(),
-  new MongoModule(),
-  new MongoStorageModule(),
-  new MySQLModule(),
-  new PostgresModule(),
-  new PostgresStorageModule()
-]);
+moduleManager.register([new CoreModule()]);
 // This is a bit of a hack. Commands such as the teardown command or even migrations might
 // want access to the ModuleManager in order to use modules
 container.register(core.ModuleManager, moduleManager);

service/src/runners/server.ts

@@ -1,6 +1,8 @@
 import { container, logger } from '@powersync/lib-services-framework';
 import * as core from '@powersync/service-core';
+
 import { logBooting } from '../util/version.js';
+import { DYNAMIC_MODULES } from '../util/modules.js';
 
 /**
  * Starts an API server
@@ -9,12 +11,18 @@ export async function startServer(runnerConfig: core.utils.RunnerConfig) {
   logBooting('API Container');
 
   const config = await core.utils.loadConfig(runnerConfig);
+
+  const moduleManager = container.getImplementation(core.modules.ModuleManager);
+  const modules = await core.loadModules(config, DYNAMIC_MODULES);
+  if (modules.length > 0) {
+    moduleManager.register(modules);
+  }
+
   const serviceContext = new core.system.ServiceContextContainer({
     serviceMode: core.system.ServiceContextMode.API,
     configuration: config
   });
 
-  const moduleManager = container.getImplementation(core.modules.ModuleManager);
   await moduleManager.initialize(serviceContext);
 
   logger.info('Starting service...');

service/src/runners/stream-worker.ts

@@ -1,6 +1,8 @@
 import { container, logger } from '@powersync/lib-services-framework';
 import * as core from '@powersync/service-core';
+
 import { logBooting } from '../util/version.js';
+import { DYNAMIC_MODULES } from '../util/modules.js';
 
 /**
  * Configures the replication portion on a {@link serviceContext}
@@ -20,15 +22,20 @@ export const startStreamRunner = async (runnerConfig: core.utils.RunnerConfig) =
   logBooting('Replication Container');
 
   const config = await core.utils.loadConfig(runnerConfig);
+
+  const moduleManager = container.getImplementation(core.modules.ModuleManager);
+  const modules = await core.loadModules(config, DYNAMIC_MODULES);
+  if (modules.length > 0) {
+    moduleManager.register(modules);
+  }
+
   // Self-hosted version allows for automatic migrations
   const serviceContext = new core.system.ServiceContextContainer({
     serviceMode: core.system.ServiceContextMode.SYNC,
     configuration: config
   });
-
   registerReplicationServices(serviceContext);
 
-  const moduleManager = container.getImplementation(core.modules.ModuleManager);
   await moduleManager.initialize(serviceContext);
 
   // Ensure automatic migrations

service/src/runners/unified-runner.ts

@@ -3,6 +3,7 @@ import * as core from '@powersync/service-core';
 
 import { logBooting } from '../util/version.js';
 import { registerReplicationServices } from './stream-worker.js';
+import { DYNAMIC_MODULES } from '../util/modules.js';
 
 /**
  * Starts an API server
@@ -11,14 +12,19 @@ export const startUnifiedRunner = async (runnerConfig: core.utils.RunnerConfig)
   logBooting('Unified Container');
 
   const config = await core.utils.loadConfig(runnerConfig);
+
+  const moduleManager = container.getImplementation(core.modules.ModuleManager);
+  const modules = await core.loadModules(config, DYNAMIC_MODULES);
+  if (modules.length > 0) {
+    moduleManager.register(modules);
+  }
+
   const serviceContext = new core.system.ServiceContextContainer({
     serviceMode: core.system.ServiceContextMode.UNIFIED,
     configuration: config
   });
-
   registerReplicationServices(serviceContext);
 
-  const moduleManager = container.getImplementation(core.modules.ModuleManager);
   await moduleManager.initialize(serviceContext);
 
   await core.migrations.ensureAutomaticMigrations({

service/src/util/modules.ts

@@ -0,0 +1,15 @@
+import * as core from '@powersync/service-core';
+
+export const DYNAMIC_MODULES: core.ModuleLoaders = {
+  connection: {
+    mongodb: () => import('@powersync/service-module-mongodb').then((module) => new module.MongoModule()),
+    mysql: () => import('@powersync/service-module-mysql').then((module) => new module.MySQLModule()),
+    postgresql: () => import('@powersync/service-module-postgres').then((module) => new module.PostgresModule())
+  },
+  storage: {
+    mongodb: () =>
+      import('@powersync/service-module-mongodb-storage').then((module) => new module.MongoStorageModule()),
+    postgresql: () =>
+      import('@powersync/service-module-postgres-storage').then((module) => new module.PostgresStorageModule())
+  }
+};