diff --git a/packages/mongodb-log-writer/src/mongo-log-manager.spec.ts b/packages/mongodb-log-writer/src/mongo-log-manager.spec.ts index b4652864..eb63b50b 100644 --- a/packages/mongodb-log-writer/src/mongo-log-manager.spec.ts +++ b/packages/mongodb-log-writer/src/mongo-log-manager.spec.ts @@ -1,6 +1,7 @@ import { MongoLogManager, mongoLogId } from '.'; import { ObjectId } from 'bson'; import { once } from 'events'; +import type { Stats, Dir } from 'fs'; import { promises as fs } from 'fs'; import path from 'path'; import os from 'os'; @@ -27,6 +28,7 @@ describe('MongoLogManager', function () { }); afterEach(async function () { await fs.rmdir(directory, { recursive: true }); + sinon.restore(); }); it('allows creating and writing to log files', async function () { @@ -86,6 +88,19 @@ describe('MongoLogManager', function () { } }); + const getFilesState = async (paths: string[]) => { + return ( + await Promise.all( + paths.map((path) => + fs.stat(path).then( + () => 1, + () => 0 + ) + ) + ) + ).join(''); + }; + it('cleans up least recent log files when requested', async function () { const manager = new MongoLogManager({ directory, @@ -106,21 +121,256 @@ describe('MongoLogManager', function () { paths.unshift(filename); } - const getFiles = async () => { - return ( - await Promise.all( - paths.map((path) => - fs.stat(path).then( - () => 1, - () => 0 - ) - ) - ) - ).join(''); - }; - expect(await getFiles()).to.equal('1111111111'); + expect(await getFilesState(paths)).to.equal('1111111111'); await manager.cleanupOldLogFiles(); - expect(await getFiles()).to.equal('0000011111'); + expect(await getFilesState(paths)).to.equal('0000011111'); + }); + + it('if fs.stat fails, it errors and is not considered towards the logs limit', async function () { + const manager = new MongoLogManager({ + directory, + retentionDays, + retentionGB: 3, + onwarn, + onerror, + }); + + const offset = Math.floor(Date.now() / 1000); + + const faultyFile = path.join( + directory, + ObjectId.createFromTime(offset - 10).toHexString() + '_log' + ); + await fs.writeFile(faultyFile, ''); + + const faultyFileError = new Error('test error'); + + const validFiles: string[] = []; + // Create 5 valid files. + for (let i = 5; i >= 0; i--) { + const filename = path.join( + directory, + ObjectId.createFromTime(offset - i).toHexString() + '_log' + ); + await fs.writeFile(filename, ''); + validFiles.push(filename); + } + + expect(onerror).not.called; + + const fsStatStub = sinon.stub(fs, 'stat'); + + fsStatStub.resolves({ + size: 1024 * 1024 * 1024, + } as Stats); + fsStatStub.withArgs(faultyFile).rejects(faultyFileError); + + await manager.cleanupOldLogFiles(); + + expect(onerror).calledOnceWithExactly(faultyFileError, faultyFile); + + // fs.stat is stubbed so getFilesState will not be accurate. + const leftoverFiles = (await fs.readdir(directory)) + .sort() + .map((file) => path.join(directory, file)); + + expect(leftoverFiles).to.have.lengthOf(4); + expect(leftoverFiles).deep.equals([faultyFile, ...validFiles.slice(3)]); + }); + + it('cleans up least recent log files when over a storage limit', async function () { + const manager = new MongoLogManager({ + directory, + retentionDays, + maxLogFileCount: 1000, + // 6 KB + retentionGB: 6 / 1024 / 1024, + onwarn, + onerror, + }); + + const paths: string[] = []; + const offset = Math.floor(Date.now() / 1000); + + // Create 10 files of 1 KB each. + for (let i = 0; i < 10; i++) { + const filename = path.join( + directory, + ObjectId.createFromTime(offset - i).toHexString() + '_log' + ); + await fs.writeFile(filename, '0'.repeat(1024)); + paths.unshift(filename); + } + + expect(await getFilesState(paths)).to.equal('1111111111'); + await manager.cleanupOldLogFiles(); + expect(await getFilesState(paths)).to.equal('0000111111'); + }); + + describe('with a random file order', function () { + let paths: string[] = []; + const times = [92, 90, 1, 2, 3, 91]; + + beforeEach(async function () { + const fileNames: string[] = []; + paths = []; + const offset = Math.floor(Date.now() / 1000); + + for (const time of times) { + const fileName = + ObjectId.createFromTime(offset - time).toHexString() + '_log'; + const fullPath = path.join(directory, fileName); + await fs.writeFile(fullPath, '0'.repeat(1024)); + fileNames.push(fileName); + paths.push(fullPath); + } + + sinon.replace(fs, 'opendir', async () => + Promise.resolve({ + [Symbol.asyncIterator]: function* () { + for (const fileName of fileNames) { + yield { + name: fileName, + isFile: () => true, + }; + } + }, + } as unknown as Dir) + ); + }); + + it('cleans up in the expected order with maxLogFileCount', async function () { + const manager = new MongoLogManager({ + directory, + retentionDays, + maxLogFileCount: 3, + onwarn, + onerror, + }); + + expect(await getFilesState(paths)).to.equal('111111'); + + await manager.cleanupOldLogFiles(); + + expect(await getFilesState(paths)).to.equal('001110'); + }); + + it('cleans up in the expected order with retentionGB', async function () { + const manager = new MongoLogManager({ + directory, + retentionDays, + retentionGB: 3 / 1024 / 1024, + onwarn, + onerror, + }); + + expect(await getFilesState(paths)).to.equal('111111'); + + await manager.cleanupOldLogFiles(); + + expect(await getFilesState(paths)).to.equal('001110'); + }); + }); + + describe('with multiple log retention settings', function () { + it('with retention days, file count, and max size maintains all conditions', async function () { + const manager = new MongoLogManager({ + directory, + retentionDays: 1, + maxLogFileCount: 3, + retentionGB: 2 / 1024 / 1024, + onwarn, + onerror, + }); + + const paths: string[] = []; + + // Create 4 files which are all older than 1 day and 4 which are from today. + for (let i = 0; i < 4; i++) { + const today = Math.floor(Date.now() / 1000); + const yesterday = today - 25 * 60 * 60; + const todayFile = path.join( + directory, + ObjectId.createFromTime(today - i).toHexString() + '_log' + ); + await fs.writeFile(todayFile, '0'.repeat(1024)); + + const yesterdayFile = path.join( + directory, + ObjectId.createFromTime(yesterday - i).toHexString() + '_log' + ); + await fs.writeFile(yesterdayFile, '0'.repeat(1024)); + + paths.unshift(todayFile); + paths.unshift(yesterdayFile); + } + + expect(await getFilesState(paths)).to.equal('11111111'); + + await manager.cleanupOldLogFiles(); + + // All yesterdays files, 2 of today's files should be deleted. + // (because of file count and file size) + expect(await getFilesState(paths)).to.equal('00000101'); + }); + + it('with low GB but high file count maintains both conditions', async function () { + const manager = new MongoLogManager({ + directory, + retentionDays, + maxLogFileCount: 3, + // 2 KB, so 2 files + retentionGB: 2 / 1024 / 1024, + onwarn, + onerror, + }); + + const paths: string[] = []; + const offset = Math.floor(Date.now() / 1000); + + // Create 10 files of 1 KB each. + for (let i = 0; i < 10; i++) { + const filename = path.join( + directory, + ObjectId.createFromTime(offset - i).toHexString() + '_log' + ); + await fs.writeFile(filename, '0'.repeat(1024)); + paths.unshift(filename); + } + + expect(await getFilesState(paths)).to.equal('1111111111'); + await manager.cleanupOldLogFiles(); + expect(await getFilesState(paths)).to.equal('0000000011'); + }); + + it('with high GB but low file count maintains both conditions', async function () { + const manager = new MongoLogManager({ + directory, + retentionDays, + maxLogFileCount: 2, + // 3 KB, so 3 files + retentionGB: 3 / 1024 / 1024, + onwarn, + onerror, + }); + + const paths: string[] = []; + const offset = Math.floor(Date.now() / 1000); + + // Create 10 files of 1 KB each. + for (let i = 0; i < 10; i++) { + const filename = path.join( + directory, + ObjectId.createFromTime(offset - i).toHexString() + '_log' + ); + await fs.writeFile(filename, '0'.repeat(1024)); + paths.unshift(filename); + } + + expect(await getFilesState(paths)).to.equal('1111111111'); + await manager.cleanupOldLogFiles(); + expect(await getFilesState(paths)).to.equal('0000000011'); + }); }); it('cleaning up old log files is a no-op by default', async function () { diff --git a/packages/mongodb-log-writer/src/mongo-log-manager.ts b/packages/mongodb-log-writer/src/mongo-log-manager.ts index 487c6a2d..180cf517 100644 --- a/packages/mongodb-log-writer/src/mongo-log-manager.ts +++ b/packages/mongodb-log-writer/src/mongo-log-manager.ts @@ -17,9 +17,11 @@ interface MongoLogOptions { retentionDays: number; /** The maximal number of log files which are kept. */ maxLogFileCount?: number; - /** A handler for warnings related to a specific filesystem path. */ - onerror: (err: Error, path: string) => unknown | Promise; + /** The maximal size of log files which are kept. */ + retentionGB?: number; /** A handler for errors related to a specific filesystem path. */ + onerror: (err: Error, path: string) => unknown | Promise; + /** A handler for warnings related to a specific filesystem path. */ onwarn: (err: Error, path: string) => unknown | Promise; } @@ -35,6 +37,17 @@ export class MongoLogManager { this._options = options; } + private async deleteFile(path: string): Promise { + try { + await fs.unlink(path); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (err: any) { + if (err?.code !== 'ENOENT') { + this._options.onerror(err as Error, path); + } + } + } + /** Clean up log files older than `retentionDays`. */ async cleanupOldLogFiles(maxDurationMs = 5_000): Promise { const dir = this._options.directory; @@ -54,8 +67,11 @@ export class MongoLogManager { const leastRecentFileHeap = new Heap<{ fileTimestamp: number; fullPath: string; + fileSize: number | undefined; }>((a, b) => a.fileTimestamp - b.fileTimestamp); + let usedStorageSize = this._options.retentionGB ? 0 : -Infinity; + for await (const dirent of dirHandle) { // Cap the overall time spent inside this function. Consider situations like // a large number of machines using a shared network-mounted $HOME directory @@ -67,32 +83,57 @@ export class MongoLogManager { const { id } = /^(?[a-f0-9]{24})_log(\.gz)?$/i.exec(dirent.name)?.groups ?? {}; if (!id) continue; + const fileTimestamp = +new ObjectId(id).getTimestamp(); const fullPath = path.join(dir, dirent.name); - let toDelete: string | undefined; // If the file is older than expected, delete it. If the file is recent, // add it to the list of seen files, and if that list is too large, remove // the least recent file we've seen so far. if (fileTimestamp < deletionCutoffTimestamp) { - toDelete = fullPath; - } else if (this._options.maxLogFileCount) { - leastRecentFileHeap.push({ fullPath, fileTimestamp }); - if (leastRecentFileHeap.size() > this._options.maxLogFileCount) { - toDelete = leastRecentFileHeap.pop()?.fullPath; - } + await this.deleteFile(fullPath); + continue; } - if (!toDelete) continue; - try { - await fs.unlink(toDelete); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } catch (err: any) { - if (err?.code !== 'ENOENT') { - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - this._options.onerror(err, fullPath); + let fileSize: number | undefined; + if (this._options.retentionGB) { + try { + fileSize = (await fs.stat(fullPath)).size; + usedStorageSize += fileSize; + } catch (err) { + this._options.onerror(err as Error, fullPath); + continue; } } + + if (this._options.maxLogFileCount || this._options.retentionGB) { + leastRecentFileHeap.push({ fullPath, fileTimestamp, fileSize }); + } + + if ( + this._options.maxLogFileCount && + leastRecentFileHeap.size() > this._options.maxLogFileCount + ) { + const toDelete = leastRecentFileHeap.pop(); + if (!toDelete) continue; + await this.deleteFile(toDelete.fullPath); + usedStorageSize -= toDelete.fileSize ?? 0; + } + } + + if (this._options.retentionGB) { + const storageSizeLimit = this._options.retentionGB * 1024 * 1024 * 1024; + + for (const file of leastRecentFileHeap) { + if (Date.now() - deletionStartTimestamp > maxDurationMs) break; + + if (usedStorageSize <= storageSizeLimit) break; + + if (!file.fileSize) continue; + + await this.deleteFile(file.fullPath); + usedStorageSize -= file.fileSize; + } } }