diff --git a/.changeset/perfect-plants-report.md b/.changeset/perfect-plants-report.md new file mode 100644 index 000000000..ac33e340e --- /dev/null +++ b/.changeset/perfect-plants-report.md @@ -0,0 +1,5 @@ +--- +'@powersync/service-image': minor +--- + +Dynamically load connection modules for reduced memory usage diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c2a8e3604..50a7b931d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,6 +19,7 @@ jobs: uses: actions/checkout@v5 - name: Login to Docker Hub + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -48,6 +49,7 @@ jobs: - uses: actions/checkout@v5 - name: Login to Docker Hub + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -95,6 +97,7 @@ jobs: - uses: actions/checkout@v5 - name: Login to Docker Hub + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -171,6 +174,7 @@ jobs: - uses: actions/checkout@v5 - name: Login to Docker Hub + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -244,6 +248,7 @@ jobs: - uses: actions/checkout@v4 - name: Login to Docker Hub + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} diff --git a/packages/service-core/src/modules/loader.ts b/packages/service-core/src/modules/loader.ts new file mode 100644 index 000000000..db7799053 --- /dev/null +++ b/packages/service-core/src/modules/loader.ts @@ -0,0 +1,47 @@ +import { ResolvedPowerSyncConfig } from '../util/util-index.js'; +import { AbstractModule } from './AbstractModule.js'; + +interface DynamicModuleMap { + [key: string]: () => Promise; +} + +export interface ModuleLoaders { + storage: DynamicModuleMap; + connection: DynamicModuleMap; +} +/** + * Utility function to dynamically load and instantiate modules. + */ +export async function loadModules(config: ResolvedPowerSyncConfig, loaders: ModuleLoaders) { + const requiredConnections = [...new Set(config.connections?.map((connection) => connection.type) || [])]; + const missingConnectionModules: string[] = []; + const modulePromises: Promise[] = []; + + // 1. Map connection types to their module loading promises making note of any + // missing connection types. + requiredConnections.forEach((connectionType) => { + const modulePromise = loaders.connection[connectionType]; + if (modulePromise !== undefined) { + modulePromises.push(modulePromise()); + } else { + missingConnectionModules.push(connectionType); + } + }); + + // Fail if any connection types are not found. + if (missingConnectionModules.length > 0) { + throw new Error(`Invalid connection types: "${[...missingConnectionModules].join(', ')}"`); + } + + if (loaders.storage[config.storage.type] !== undefined) { + modulePromises.push(loaders.storage[config.storage.type]()); + } else { + throw new Error(`Invalid storage type: "${config.storage.type}"`); + } + + // 2. Dynamically import and instantiate module classes and resolve all promises + // raising errors if any modules could not be imported. + const moduleInstances = await Promise.all(modulePromises); + + return moduleInstances; +} diff --git a/packages/service-core/src/modules/modules-index.ts b/packages/service-core/src/modules/modules-index.ts index 1b32990e8..2bea3bf4e 100644 --- a/packages/service-core/src/modules/modules-index.ts +++ b/packages/service-core/src/modules/modules-index.ts @@ -1,2 +1,3 @@ export * from './ModuleManager.js'; export * from './AbstractModule.js'; +export * from './loader.js'; diff --git a/packages/service-core/test/src/module-loader.test.ts b/packages/service-core/test/src/module-loader.test.ts new file mode 100644 index 000000000..102086889 --- /dev/null +++ b/packages/service-core/test/src/module-loader.test.ts @@ -0,0 +1,102 @@ +import { AbstractModule, loadModules, ServiceContextContainer, TearDownOptions } from '@/index.js'; +import { describe, expect, it, vi } from 'vitest'; + +interface MockConfig { + connections?: { type: string }[]; + storage: { type: string }; +} + +class MockMySQLModule extends AbstractModule { + constructor() { + super({ name: 'MySQLModule' }); + } + async initialize(context: ServiceContextContainer): Promise {} + async teardown(options: TearDownOptions): Promise {} +} +class MockPostgresModule extends AbstractModule { + constructor() { + super({ name: 'PostgresModule' }); + } + async initialize(context: ServiceContextContainer): Promise {} + async teardown(options: TearDownOptions): Promise {} +} +class MockPostgresStorageModule extends AbstractModule { + constructor() { + super({ name: 'PostgresStorageModule' }); + } + async initialize(context: ServiceContextContainer): Promise {} + async teardown(options: TearDownOptions): Promise {} +} +const mockLoaders = { + connection: { + mysql: async () => { + return new MockMySQLModule(); + }, + postgresql: async () => { + return new MockPostgresModule(); + } + }, + storage: { + postgresql: async () => { + return new MockPostgresStorageModule(); + } + } +}; + +describe('module loader', () => { + it('should load all modules defined in connections and storage', async () => { + const config: MockConfig = { + connections: [{ type: 'mysql' }, { type: 'postgresql' }], + storage: { type: 'postgresql' } + }; + + const modules = await loadModules(config as any, mockLoaders); + + expect(modules.length).toBe(3); + expect(modules[0]).toBeInstanceOf(MockMySQLModule); + expect(modules[1]).toBeInstanceOf(MockPostgresModule); + expect(modules[2]).toBeInstanceOf(MockPostgresStorageModule); + }); + + it('should handle duplicate connection types (e.g., mysql used twice)', async () => { + const config: MockConfig = { + connections: [{ type: 'mysql' }, { type: 'postgresql' }, { type: 'mysql' }], // mysql duplicated + storage: { type: 'postgresql' } + }; + + const modules = await loadModules(config as any, mockLoaders); + + // Expect 3 modules: mysql, postgresql, postgresql-storage + expect(modules.length).toBe(3); + expect(modules.filter((m) => m instanceof MockMySQLModule).length).toBe(1); + expect(modules.filter((m) => m instanceof MockPostgresModule).length).toBe(1); + expect(modules.filter((m) => m instanceof MockPostgresStorageModule).length).toBe(1); + }); + + it('should throw an error if any modules are not found in ModuleMap', async () => { + const config: MockConfig = { + connections: [{ type: 'mysql' }, { type: 'redis' }], + storage: { type: 'postgresql' } + }; + + await expect(loadModules(config as any, mockLoaders)).rejects.toThrowError(); + }); + + it('should throw an error if one dynamic connection module import fails', async () => { + const config: MockConfig = { + connections: [{ type: 'mysql' }], + storage: { type: 'postgresql' } + }; + + const loaders = { + connection: { + mysql: async () => { + throw new Error('Failed to load MySQL module'); + } + }, + storage: mockLoaders.storage + }; + + await expect(loadModules(config as any, loaders)).rejects.toThrowError('Failed to load MySQL module'); + }); +}); diff --git a/service/package.json b/service/package.json index 6baee3abb..5c63539e9 100644 --- a/service/package.json +++ b/service/package.json @@ -28,4 +28,4 @@ "npm-check-updates": "^16.14.4", "ts-node": "^10.9.1" } -} +} \ No newline at end of file diff --git a/service/src/entry.ts b/service/src/entry.ts index 61b943f17..9d2bac42a 100644 --- a/service/src/entry.ts +++ b/service/src/entry.ts @@ -2,11 +2,6 @@ import { container, ContainerImplementation } from '@powersync/lib-services-fram import * as core from '@powersync/service-core'; import { CoreModule } from '@powersync/service-module-core'; -import { MongoModule } from '@powersync/service-module-mongodb'; -import { MongoStorageModule } from '@powersync/service-module-mongodb-storage'; -import { MySQLModule } from '@powersync/service-module-mysql'; -import { PostgresModule } from '@powersync/service-module-postgres'; -import { PostgresStorageModule } from '@powersync/service-module-postgres-storage'; import { startServer } from './runners/server.js'; import { startStreamRunner } from './runners/stream-worker.js'; import { startUnifiedRunner } from './runners/unified-runner.js'; @@ -17,14 +12,7 @@ container.registerDefaults(); container.register(ContainerImplementation.REPORTER, createSentryReporter()); const moduleManager = new core.modules.ModuleManager(); -moduleManager.register([ - new CoreModule(), - new MongoModule(), - new MongoStorageModule(), - new MySQLModule(), - new PostgresModule(), - new PostgresStorageModule() -]); +moduleManager.register([new CoreModule()]); // This is a bit of a hack. Commands such as the teardown command or even migrations might // want access to the ModuleManager in order to use modules container.register(core.ModuleManager, moduleManager); diff --git a/service/src/runners/server.ts b/service/src/runners/server.ts index 5876a6977..6484cfaab 100644 --- a/service/src/runners/server.ts +++ b/service/src/runners/server.ts @@ -1,6 +1,8 @@ import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; + import { logBooting } from '../util/version.js'; +import { DYNAMIC_MODULES } from '../util/modules.js'; /** * Starts an API server @@ -9,12 +11,18 @@ export async function startServer(runnerConfig: core.utils.RunnerConfig) { logBooting('API Container'); const config = await core.utils.loadConfig(runnerConfig); + + const moduleManager = container.getImplementation(core.modules.ModuleManager); + const modules = await core.loadModules(config, DYNAMIC_MODULES); + if (modules.length > 0) { + moduleManager.register(modules); + } + const serviceContext = new core.system.ServiceContextContainer({ serviceMode: core.system.ServiceContextMode.API, configuration: config }); - const moduleManager = container.getImplementation(core.modules.ModuleManager); await moduleManager.initialize(serviceContext); logger.info('Starting service...'); diff --git a/service/src/runners/stream-worker.ts b/service/src/runners/stream-worker.ts index e9197a376..8d4ebaf74 100644 --- a/service/src/runners/stream-worker.ts +++ b/service/src/runners/stream-worker.ts @@ -1,6 +1,8 @@ import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; + import { logBooting } from '../util/version.js'; +import { DYNAMIC_MODULES } from '../util/modules.js'; /** * Configures the replication portion on a {@link serviceContext} @@ -20,15 +22,20 @@ export const startStreamRunner = async (runnerConfig: core.utils.RunnerConfig) = logBooting('Replication Container'); const config = await core.utils.loadConfig(runnerConfig); + + const moduleManager = container.getImplementation(core.modules.ModuleManager); + const modules = await core.loadModules(config, DYNAMIC_MODULES); + if (modules.length > 0) { + moduleManager.register(modules); + } + // Self-hosted version allows for automatic migrations const serviceContext = new core.system.ServiceContextContainer({ serviceMode: core.system.ServiceContextMode.SYNC, configuration: config }); - registerReplicationServices(serviceContext); - const moduleManager = container.getImplementation(core.modules.ModuleManager); await moduleManager.initialize(serviceContext); // Ensure automatic migrations diff --git a/service/src/runners/unified-runner.ts b/service/src/runners/unified-runner.ts index c008286c9..88b93c1f5 100644 --- a/service/src/runners/unified-runner.ts +++ b/service/src/runners/unified-runner.ts @@ -3,6 +3,7 @@ import * as core from '@powersync/service-core'; import { logBooting } from '../util/version.js'; import { registerReplicationServices } from './stream-worker.js'; +import { DYNAMIC_MODULES } from '../util/modules.js'; /** * Starts an API server @@ -11,14 +12,19 @@ export const startUnifiedRunner = async (runnerConfig: core.utils.RunnerConfig) logBooting('Unified Container'); const config = await core.utils.loadConfig(runnerConfig); + + const moduleManager = container.getImplementation(core.modules.ModuleManager); + const modules = await core.loadModules(config, DYNAMIC_MODULES); + if (modules.length > 0) { + moduleManager.register(modules); + } + const serviceContext = new core.system.ServiceContextContainer({ serviceMode: core.system.ServiceContextMode.UNIFIED, configuration: config }); - registerReplicationServices(serviceContext); - const moduleManager = container.getImplementation(core.modules.ModuleManager); await moduleManager.initialize(serviceContext); await core.migrations.ensureAutomaticMigrations({ diff --git a/service/src/util/modules.ts b/service/src/util/modules.ts new file mode 100644 index 000000000..5ef36858c --- /dev/null +++ b/service/src/util/modules.ts @@ -0,0 +1,15 @@ +import * as core from '@powersync/service-core'; + +export const DYNAMIC_MODULES: core.ModuleLoaders = { + connection: { + mongodb: () => import('@powersync/service-module-mongodb').then((module) => new module.MongoModule()), + mysql: () => import('@powersync/service-module-mysql').then((module) => new module.MySQLModule()), + postgresql: () => import('@powersync/service-module-postgres').then((module) => new module.PostgresModule()) + }, + storage: { + mongodb: () => + import('@powersync/service-module-mongodb-storage').then((module) => new module.MongoStorageModule()), + postgresql: () => + import('@powersync/service-module-postgres-storage').then((module) => new module.PostgresStorageModule()) + } +};