CI: Add memory usage regression monitoring for pull requests (#1415)

.github/workflows/benchmark.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   benchmark:
-    name: Time benchmark
+    name: Time and memory usage benchmark
     runs-on: ubuntu-latest
 
     steps:
@@ -23,13 +23,19 @@ jobs:
         with:
           node-version: '15'
 
-      - name: Run pull request benchmark
-        run: cd pr && npm install && node test/benchmarks/benchmark.js > benchmarks.txt && cat benchmarks.txt
+      - name: Run pull request time benchmark
+        run: cd pr && npm install && npm run --silent benchmark-time > benchmarks.txt && cat benchmarks.txt
 
-      - name: Run benchmark on master (baseline)
-        run: cd master && npm install && node test/benchmarks/benchmark.js > benchmarks.txt && cat benchmarks.txt
+      - name: Run pull request memory usage benchmark
+        run: cd pr && npm run --silent benchmark-memory-usage > memory_usage.txt && cat memory_usage.txt
 
-      - name: Compare benchmark result
+      - name: Run time benchmark on master (baseline)
+        run: cd master && npm install && npm run --silent benchmark-time > benchmarks.txt && cat benchmarks.txt
+
+      - name: Run memory usage benchmark on master (baseline)
+        run: cd master && npm run --silent benchmark-memory-usage > memory_usage.txt && cat memory_usage.txt
+
+      - name: Compare time benchmark result
         uses: openpgpjs/github-action-pull-request-benchmark@v1
         with:
           tool: 'benchmarkjs'
@@ -43,3 +49,16 @@ jobs:
           # fail workdlow if 1.5 times slower
           fail-threshold: '150%'
           fail-on-alert: true
+
+      - name: Compare memory usage benchmark result
+        uses: openpgpjs/github-action-pull-request-benchmark@v1
+        with:
+          tool: 'raw'
+          name: 'memory usage benchmark'
+          pr-benchmark-file-path: pr/memory_usage.txt
+          base-benchmark-file-path: master/memory_usage.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          alert-threshold: '102%'
+          comment-on-alert: true
+          fail-threshold: '110%'
+          fail-on-alert: true

package.json

@@ -39,6 +39,8 @@
     "prepare": "npm run build",
     "test": "mocha --require esm --timeout 120000 test/unittests.js",
     "test-type-definitions": "tsc test/typescript/definitions.ts && node test/typescript/definitions.js",
+    "benchmark-time": "node test/benchmarks/time.js",
+    "benchmark-memory-usage": "node --require esm test/benchmarks/memory_usage.js",
     "start": "http-server",
     "prebrowsertest": "npm run build-test",
     "browsertest": "npm start -- -o test/unittests.html",

test/benchmarks/memory_usage.js

@@ -0,0 +1,219 @@
+/* eslint-disable no-console */
+const assert = require('assert');
+const stream = require('@openpgp/web-stream-tools');
+const path = require('path');
+const { writeFileSync, unlinkSync } = require('fs');
+const { fork } = require('child_process');
+const openpgp = require('../..');
+
+/**
+ * Benchmark max memory usage recorded during execution of the given function.
+ * This spawns a new v8 instance and runs the code there in isolation, to avoid interference between tests.
+ * @param {Funtion} function to benchmark (can be async)
+ * @returns {NodeJS.MemoryUsage} memory usage snapshot with max RSS (sizes in bytes)
+ */
+const benchmark = async function(fn) {
+  const tmpFileName = path.join(__dirname, 'tmp.js');
+  // the code to execute must be written to a file
+  writeFileSync(tmpFileName, `
+const assert = require('assert');
+const stream = require('@openpgp/web-stream-tools');
+const openpgp = require('../..');
+let maxMemoryComsumption;
+let activeSampling = false;
+
+function sampleOnce() {
+  const memUsage = process.memoryUsage();
+  if (!maxMemoryComsumption || memUsage.rss > maxMemoryComsumption.rss) {
+    maxMemoryComsumption = memUsage;
+  }
+}
+
+function samplePeriodically() {
+  setImmediate(() => {
+    sampleOnce();
+    activeSampling && samplePeriodically();
+  });
+}
+
+// main body
+(async () => {
+  maxMemoryComsumption = null;
+  activeSampling = true;
+  samplePeriodically();
+  await (${fn.toString()})();
+  // setImmediate is run at the end of the event loop, so we need to manually collect the latest sample
+  sampleOnce();
+  process.send(maxMemoryComsumption);
+  process.exit(); // child process doesn't exit otherwise
+})();
+`);
+
+  const maxMemoryComsumption = await new Promise((resolve, reject) => {
+    const child = fork(tmpFileName);
+    child.on('message', function (message) {
+      resolve(message);
+    });
+    child.on('error', function (err) {
+      reject(err);
+    });
+  });
+
+  unlinkSync(tmpFileName);
+  return maxMemoryComsumption;
+};
+
+const onError = err => {
+  console.error('The memory benchmark tests failed by throwing the following error:');
+  console.error(err);
+  // eslint-disable-next-line no-process-exit
+  process.exit(1);
+};
+
+class MemoryBenchamrkSuite {
+  constructor() {
+    this.tests = [];
+  }
+
+  add(name, fn) {
+    this.tests.push({ name, fn });
+  }
+
+  async run() {
+    const stats = [];
+    for (const { name, fn } of this.tests) {
+      const memoryUsage = await benchmark(fn).catch(onError);
+      // convert values to MB
+      Object.entries(memoryUsage).forEach(([name, value]) => {
+        memoryUsage[name] = (value / 1024 / 1024).toFixed(2);
+      });
+      const { rss, ...usageDetails } = memoryUsage;
+      // raw entry format accepted by github-action-pull-request-benchmark
+      stats.push({
+        name,
+        value: rss,
+        range: Object.entries(usageDetails).map(([name, value]) => `${name}: ${value}`).join(', '),
+        unit: 'MB',
+        biggerIsBetter: false
+      });
+    }
+    return stats;
+  }
+}
+
+/**
+ * Memory usage tests.
+ * All the necessary variables must be declared inside the test function.
+ */
+(async () => {
+  const suite = new MemoryBenchamrkSuite();
+
+  suite.add('empty test (baseline)', () => {});
+
+  suite.add('openpgp.encrypt/decrypt (CFB, binary)', async () => {
+    const passwords = 'password';
+    const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
+    const plaintextMessage = await openpgp.createMessage({ binary: new Uint8Array(1000000).fill(1) });
+
+    const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
+    const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
+    assert.ok(encryptedMessage.packets[1] instanceof openpgp.SymEncryptedIntegrityProtectedDataPacket);
+    await openpgp.decrypt({ message: encryptedMessage, passwords, config });
+  });
+
+  suite.add('openpgp.encrypt/decrypt (CFB, text)', async () => {
+    const passwords = 'password';
+    const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
+    const plaintextMessage = await openpgp.createMessage({ text: 'a'.repeat(10000000 / 2) }); // two bytes per character
+
+    const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
+    const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
+    assert.ok(encryptedMessage.packets[1] instanceof openpgp.SymEncryptedIntegrityProtectedDataPacket);
+    await openpgp.decrypt({ message: encryptedMessage, passwords, config });
+  });
+
+  suite.add('openpgp.encrypt/decrypt (AEAD, binary)', async () => {
+    const passwords = 'password';
+    const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
+    const plaintextMessage = await openpgp.createMessage({ binary: new Uint8Array(1000000).fill(1) });
+
+    const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
+    const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
+    assert.ok(encryptedMessage.packets[1] instanceof openpgp.AEADEncryptedDataPacket);
+    await openpgp.decrypt({ message: encryptedMessage, passwords, config });
+  });
+
+  suite.add('openpgp.encrypt/decrypt (AEAD, text)', async () => {
+    const passwords = 'password';
+    const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
+    const plaintextMessage = await openpgp.createMessage({ text: 'a'.repeat(10000000 / 2) }); // two bytes per character
+
+    const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
+    const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
+    assert.ok(encryptedMessage.packets[1] instanceof openpgp.AEADEncryptedDataPacket);
+    await openpgp.decrypt({ message: encryptedMessage, passwords, config });
+  });
+
+  // streaming tests
+  suite.add('openpgp.encrypt/decrypt (CFB, binary, with streaming)', async () => {
+    await stream.loadStreamsPonyfill();
+
+    const passwords = 'password';
+    const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
+    const plaintextMessage = await openpgp.createMessage({ binary: stream.toStream(new Uint8Array(1000000).fill(1)) });
+    assert(plaintextMessage.fromStream);
+
+    const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
+    const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
+    assert.ok(encryptedMessage.packets[1] instanceof openpgp.SymEncryptedIntegrityProtectedDataPacket);
+    const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
+    await stream.readToEnd(decryptedData);
+  });
+
+  suite.add('openpgp.encrypt/decrypt (CFB, text, with streaming)', async () => {
+    await stream.loadStreamsPonyfill();
+
+    const passwords = 'password';
+    const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
+    const plaintextMessage = await openpgp.createMessage({ text: stream.toStream('a'.repeat(10000000 / 2)) });
+    assert(plaintextMessage.fromStream);
+
+    const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
+    const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
+    assert.ok(encryptedMessage.packets[1] instanceof openpgp.SymEncryptedIntegrityProtectedDataPacket);
+    const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
+    await stream.readToEnd(decryptedData);
+  });
+
+  suite.add('openpgp.encrypt/decrypt (AEAD, binary, with streaming)', async () => {
+    await stream.loadStreamsPonyfill();
+
+    const passwords = 'password';
+    const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
+    const plaintextMessage = await openpgp.createMessage({ binary: stream.toStream(new Uint8Array(1000000).fill(1)) });
+    assert(plaintextMessage.fromStream);
+
+    const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
+    const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
+    assert.ok(encryptedMessage.packets[1] instanceof openpgp.AEADEncryptedDataPacket);
+    await openpgp.decrypt({ message: encryptedMessage, passwords, config });
+  });
+
+  suite.add('openpgp.encrypt/decrypt (AEAD, text, with streaming)', async () => {
+    await stream.loadStreamsPonyfill();
+
+    const passwords = 'password';
+    const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
+    const plaintextMessage = await openpgp.createMessage({ text: stream.toStream('a'.repeat(10000000 / 2)) });
+    assert(plaintextMessage.fromStream);
+
+    const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
+    const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
+    assert.ok(encryptedMessage.packets[1] instanceof openpgp.AEADEncryptedDataPacket);
+    await openpgp.decrypt({ message: encryptedMessage, passwords, config });
+  });
+
+  const stats = await suite.run();
+  // Print JSON stats to stdout
+  console.log(JSON.stringify(stats, null, 4));
+})();