|
| 1 | +/* eslint-disable no-console */ |
| 2 | +/* eslint-disable no-unused-expressions */ |
| 3 | +const { DummyRequestLogger } = require('../../unit/helpers'); |
| 4 | +const assert = require('assert'); |
| 5 | +const log = new DummyRequestLogger(); |
| 6 | +const helpers = require('./helpers'); |
| 7 | +const { spawn } = require('child_process'); |
| 8 | +const path = require('path'); |
| 9 | +const kms = require('../../../lib/kms/wrapper'); |
| 10 | +const { promisify } = require('util'); |
| 11 | + |
| 12 | +const BUCKET_NUMBER = 10; |
| 13 | +const OBJECT_NUMBER = 200; |
| 14 | +const TOTAL_OBJECTS = BUCKET_NUMBER * OBJECT_NUMBER; |
| 15 | + |
| 16 | +const KMS_NODES = helpers.config.kmip.transport.length; |
| 17 | +const TOTAL_OBJECTS_PER_NODE = Math.floor(TOTAL_OBJECTS / KMS_NODES); |
| 18 | + |
| 19 | +/** |
| 20 | + * 10% approximation for the number of packets per IP |
| 21 | + * As we might not have an exact match of packets and the |
| 22 | + * round robin is confined to each nodejs cluster processes |
| 23 | + */ |
| 24 | +const APPROX = Math.floor(0.1 * TOTAL_OBJECTS_PER_NODE); |
| 25 | +const EXPECTED_MIN = TOTAL_OBJECTS_PER_NODE - APPROX; |
| 26 | +const EXPECTED_MAX = TOTAL_OBJECTS_PER_NODE + APPROX; |
| 27 | + |
| 28 | +async function spawnTcpdump(port, packetCount) { |
| 29 | + const scriptPath = path.join(__dirname, 'countPacketsByIp.sh'); |
| 30 | + return new Promise((resolve, reject) => { |
| 31 | + const child = spawn( |
| 32 | + 'sudo', // run as root to allow tcpdump execution |
| 33 | + // 5m timeout as process is detached |
| 34 | + ['timeout', 300, scriptPath, port, packetCount], |
| 35 | + { |
| 36 | + // detach to not mess with the tty, it would cause issues with |
| 37 | + // \r even when using another shell and piping stdout. |
| 38 | + detached: true, |
| 39 | + stdio: ['ignore', 'pipe', 'pipe'], // ignored stdin |
| 40 | + shell: false, // no need as it's detached |
| 41 | + |
| 42 | + }, |
| 43 | + ); |
| 44 | + let stderr = ''; |
| 45 | + child.stderr.on('data', data => { |
| 46 | + stderr += data.toString(); |
| 47 | + }); |
| 48 | + |
| 49 | + /** Let a small 10ms timeout for a potential error */ |
| 50 | + let spawnTimeout; |
| 51 | + child.on('spawn', () => { |
| 52 | + spawnTimeout = setTimeout(() => { |
| 53 | + if (child.exitCode !== null || child.signalCode !== null) { |
| 54 | + const err = `countPacketsByIp.sh stopped after spawn with code ${ |
| 55 | + child.exitCode} and signal ${child.signalCode}.\nStderr: ${stderr}`; |
| 56 | + reject(new Error(err)); |
| 57 | + } else { |
| 58 | + resolve(child); |
| 59 | + } |
| 60 | + }, 10); |
| 61 | + }); |
| 62 | + child.on('error', err => { |
| 63 | + if (spawnTimeout) { |
| 64 | + clearTimeout(spawnTimeout); |
| 65 | + } |
| 66 | + reject(new Error(`${err.toString()}\nStderr: ${stderr}`)); |
| 67 | + }); |
| 68 | + |
| 69 | + child.on('close', (code, signal) => { |
| 70 | + if (code) { |
| 71 | + if (spawnTimeout) { |
| 72 | + clearTimeout(spawnTimeout); |
| 73 | + } |
| 74 | + reject(new Error( |
| 75 | + `tcpdump script closed with code ${code} and signal ${signal}.\nStderr: ${stderr}` |
| 76 | + )); |
| 77 | + } |
| 78 | + }); |
| 79 | + }); |
| 80 | +} |
| 81 | + |
| 82 | +async function stopTcpdump(tcpdump) { |
| 83 | + if (tcpdump.exitCode !== null || tcpdump.signalCode !== null) { |
| 84 | + // tcpdump already closed, no need to kill it |
| 85 | + return; |
| 86 | + } |
| 87 | + await new Promise(resolve => { |
| 88 | + tcpdump.on('close', resolve); |
| 89 | + tcpdump.kill('SIGTERM'); |
| 90 | + }); |
| 91 | +} |
| 92 | + |
| 93 | +describe(`KMS load (kmip cluster ${KMS_NODES} nodes): ${OBJECT_NUMBER |
| 94 | + } objs each in ${BUCKET_NUMBER} bkts (${TOTAL_OBJECTS} objs)`, () => { |
| 95 | + let buckets = []; |
| 96 | + let tcpdumpProcess; |
| 97 | + let stdout; |
| 98 | + let stderr; |
| 99 | + let closePromise; |
| 100 | + |
| 101 | + before(async () => { |
| 102 | + buckets = await Promise.all( |
| 103 | + new Array(BUCKET_NUMBER).fill(0).map(async (_, i) => { |
| 104 | + const Bucket = `kms-load-${i}`; |
| 105 | + const { masterKeyArn } = await helpers.createKmsKey(log); |
| 106 | + |
| 107 | + await helpers.s3.createBucket({ Bucket }).promise(); |
| 108 | + await helpers.s3.putBucketEncryption({ |
| 109 | + Bucket, |
| 110 | + ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ |
| 111 | + algo: 'aws:kms', masterKeyId: masterKeyArn }), |
| 112 | + }).promise(); |
| 113 | + |
| 114 | + return { Bucket, masterKeyArn }; |
| 115 | + })); |
| 116 | + }); |
| 117 | + |
| 118 | + after(async () => { |
| 119 | + await Promise.all(buckets.map(async ({ Bucket, masterKeyArn }) => { |
| 120 | + await helpers.cleanup(Bucket); |
| 121 | + return helpers.destroyKmsKey(masterKeyArn, log); |
| 122 | + })); |
| 123 | + await promisify(kms.client.stop.bind(kms.client))(); |
| 124 | + }); |
| 125 | + |
| 126 | + beforeEach(async () => { |
| 127 | + // tcpdump can catch more than TOTAL_OBJECTS packets because there are PSH and ACK packets |
| 128 | + // but we need to ensure it actually stops before there is no more packets |
| 129 | + // to count packets by IP |
| 130 | + tcpdumpProcess = await spawnTcpdump(5696, TOTAL_OBJECTS); |
| 131 | + stdout = ''; |
| 132 | + stderr = ''; |
| 133 | + tcpdumpProcess.stderr.on('data', data => { |
| 134 | + stderr += data.toString(); |
| 135 | + }); |
| 136 | + tcpdumpProcess.stdout.on('data', data => { |
| 137 | + stdout += data.toString(); |
| 138 | + }); |
| 139 | + closePromise = new Promise(resolve => { |
| 140 | + tcpdumpProcess.on('close', (code, signal) => |
| 141 | + resolve({ |
| 142 | + code, |
| 143 | + signal, |
| 144 | + repartition: stdout |
| 145 | + .split('\n') |
| 146 | + .filter(l => l) |
| 147 | + .map(line => { |
| 148 | + const [count, ip] = line.trim().split(' '); |
| 149 | + return { count: +count, ip }; |
| 150 | + }), |
| 151 | + }) |
| 152 | + ); |
| 153 | + }); |
| 154 | + }); |
| 155 | + |
| 156 | + afterEach(async () => { |
| 157 | + if (tcpdumpProcess) { |
| 158 | + await stopTcpdump(tcpdumpProcess); |
| 159 | + } |
| 160 | + }); |
| 161 | + |
| 162 | + async function assertRepartition(closePromise) { |
| 163 | + const { code, signal, repartition } = await closePromise; |
| 164 | + console.log('Test Details', { |
| 165 | + KMS_NODES, |
| 166 | + TOTAL_OBJECTS, |
| 167 | + TOTAL_OBJECTS_PER_NODE, |
| 168 | + APPROX, |
| 169 | + EXPECTED_MIN, |
| 170 | + EXPECTED_MAX, |
| 171 | + code, |
| 172 | + signal, |
| 173 | + stderr, |
| 174 | + repartition, |
| 175 | + }); |
| 176 | + const repartitionCount = repartition.map(({ count }) => count); |
| 177 | + assert.strictEqual(code, 0, `tcpdump script closed with code ${code} and signal ${signal}`); |
| 178 | + assert(repartition.length === KMS_NODES, `Expected ${KMS_NODES} IPs but got ${repartition.length}`); |
| 179 | + assert(repartitionCount.every(count => |
| 180 | + count >= EXPECTED_MIN && count <= EXPECTED_MAX), |
| 181 | + `Repartition counts should be around ${TOTAL_OBJECTS_PER_NODE} but got ${repartitionCount}`); |
| 182 | + } |
| 183 | + |
| 184 | + it(`should encrypt ${TOTAL_OBJECTS} times in parallel, ~${TOTAL_OBJECTS_PER_NODE} per node`, async () => { |
| 185 | + await (Promise.all( |
| 186 | + buckets.map(async ({ Bucket }) => Promise.all( |
| 187 | + new Array(OBJECT_NUMBER).fill(0).map(async (_, i) => |
| 188 | + helpers.s3.putObject({ Bucket, Key: `obj-${i}`, Body: `body-${i}` }).promise()) |
| 189 | + )) |
| 190 | + )); |
| 191 | + await assertRepartition(closePromise); |
| 192 | + }); |
| 193 | + |
| 194 | + it(`should decrypt ${TOTAL_OBJECTS} times in parallel, ~${TOTAL_OBJECTS_PER_NODE} per node`, async () => { |
| 195 | + await Promise.all( |
| 196 | + buckets.map(async ({ Bucket }) => Promise.all( |
| 197 | + new Array(OBJECT_NUMBER).fill(0).map(async (_, i) => |
| 198 | + helpers.s3.getObject({ Bucket, Key: `obj-${i}` }).promise()) |
| 199 | + )) |
| 200 | + ); |
| 201 | + await assertRepartition(closePromise); |
| 202 | + }); |
| 203 | +}); |
0 commit comments