CI: Add performance regression monitoring for pull requests (#1411)

.github/workflows/benchmark.yml

@@ -0,0 +1,45 @@
+name: Performance Regression Test
+
+on:
+  pull_request:
+    branches: [master]
+
+jobs:
+  benchmark:
+    name: Time benchmark
+    runs-on: ubuntu-latest
+
+    steps:
+      # check out pull request branch
+      - uses: actions/checkout@v2
+        with:
+          path: pr
+      # check out master branch (to compare performance)
+      - uses: actions/checkout@v2
+        with:
+          ref: master
+          path: master
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '15'
+
+      - name: Run pull request benchmark
+        run: cd pr && npm install && node test/benchmarks/benchmark.js > benchmarks.txt && cat benchmarks.txt
+
+      - name: Run benchmark on master (baseline)
+        run: cd master && npm install && node test/benchmarks/benchmark.js > benchmarks.txt && cat benchmarks.txt
+
+      - name: Compare benchmark result
+        uses: openpgpjs/github-action-pull-request-benchmark@v1
+        with:
+          tool: 'benchmarkjs'
+          name: 'time benchmark'
+          pr-benchmark-file-path: pr/benchmarks.txt
+          base-benchmark-file-path: master/benchmarks.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          # trigger alert comment if 1.3 times slower
+          alert-threshold: '130%'
+          comment-on-alert: true
+          # fail workdlow if 1.5 times slower
+          fail-threshold: '150%'
+          fail-on-alert: true

package-lock.json

@@ -662,6 +662,16 @@
       "integrity": "sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog==",
       "dev": true
     },
+    "benchmark": {
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/benchmark/-/benchmark-2.1.4.tgz",
+      "integrity": "sha1-CfPeMckWQl1JjMLuVloOvzwqVik=",
+      "dev": true,
+      "requires": {
+        "lodash": "^4.17.4",
+        "platform": "^1.3.3"
+      }
+    },
     "binary-extensions": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@@ -3923,6 +3933,12 @@
         "find-up": "^1.0.0"
       }
     },
+    "platform": {
+      "version": "1.3.6",
+      "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz",
+      "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==",
+      "dev": true
+    },
     "pluralize": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-7.0.0.tgz",

package.json

@@ -63,6 +63,7 @@
     "@rollup/plugin-replace": "^2.3.2",
     "@types/chai": "^4.2.14",
     "babel-eslint": "^10.1.0",
+    "benchmark": "^2.1.4",
     "bn.js": "^4.11.8",
     "chai": "^4.1.2",
     "chai-as-promised": "^7.1.1",

test/benchmarks/benchmark.js

@@ -0,0 +1,100 @@
+const Benchmark = require('benchmark');
+const openpgp = typeof window !== 'undefined' && window.openpgp ? window.openpgp : require('../../');
+
+const wrapAsync = func => ({
+  fn: async deferred => {
+    await func().catch(onError);
+    deferred.resolve();
+  },
+  defer: true
+});
+
+const onError = err => {
+  // eslint-disable-next-line no-console
+  console.error('The time benchmark tests failed by throwing the following error:');
+  // eslint-disable-next-line no-console
+  console.error(err);
+  // eslint-disable-next-line no-process-exit
+  process.exit(1);
+};
+
+/**
+ * Time benchmark tests.
+ * NB: each test will be run multiple times, so any input must be consumable multiple times.
+ */
+(async () => {
+  const suite = new Benchmark.Suite();
+  const { armoredKey, privateKey, publicKey, armoredEncryptedMessage, armoredSignedMessage } = await getTestData();
+
+  suite.add('openpgp.readKey', wrapAsync(async () => {
+    await openpgp.readKey({ armoredKey });
+  }));
+
+  suite.add('openpgp.readMessage', wrapAsync(async () => {
+    await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
+  }));
+
+  suite.add('openpgp.generateKey', wrapAsync(async () => {
+    await openpgp.generateKey({ userIDs: { email: 'test@test.it' } });
+  }));
+
+  suite.add('openpgp.encrypt', wrapAsync(async () => {
+    const message = await openpgp.createMessage({ text: 'plaintext' });
+    await openpgp.encrypt({ message, encryptionKeys: publicKey });
+  }));
+
+  suite.add('openpgp.sign', wrapAsync(async () => {
+    const message = await openpgp.createMessage({ text: 'plaintext' });
+    await openpgp.sign({ message, signingKeys: privateKey });
+  }));
+
+  suite.add('openpgp.decrypt', wrapAsync(async () => {
+    const message = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
+    await openpgp.decrypt({ message, decryptionKeys: privateKey });
+  }));
+
+  suite.add('openpgp.verify', wrapAsync(async () => {
+    const message = await openpgp.readMessage({ armoredMessage: armoredSignedMessage });
+    await openpgp.verify({ message, verificationKeys: publicKey, expectSigned: true });
+  }));
+
+  suite.on('cycle', event => {
+    // Output benchmark result by converting benchmark result to string
+    // eslint-disable-next-line no-console
+    console.log(String(event.target));
+  });
+
+  suite.run({ 'async': true });
+})();
+
+async function getTestData() {
+  const armoredKey = `-----BEGIN PGP PRIVATE KEY BLOCK-----
+
+xVgEYS4KIRYJKwYBBAHaRw8BAQdAOl5Ij0p8llEOLqalwRM8+YWKXELm+Zl1
+arT2orL/42MAAP9SQBdl+A/i4AtIOr33rn6OKzmXQ2EQH0xoSPJcVxX7BA5U
+zRR0ZXN0IDx0ZXN0QHRlc3QuY29tPsKMBBAWCgAdBQJhLgohBAsJBwgDFQgK
+BBYAAgECGQECGwMCHgEAIQkQ2RFo4G/cGHQWIQRL9hTrZduw8+42e1rZEWjg
+b9wYdEi3AP91NftBKXLfcMRz/g540cQ/0+ax8pvsiqFSb+Sqz87YPwEAkoYK
+8I9rVAlVABIhy/g7ZStHu/u0zsPbiquZFKoVLgPHXQRhLgohEgorBgEEAZdV
+AQUBAQdAqY5VZYX6axscpfVN3EED83T3WO3+Hzxfq31dXJXKrRkDAQgHAAD/
+an6zziN/Aw0ruIxuZTjmkYriDW34hys8F2nRR23PO6gPjsJ4BBgWCAAJBQJh
+LgohAhsMACEJENkRaOBv3Bh0FiEES/YU62XbsPPuNnta2RFo4G/cGHQjlgEA
+gbOEmauiq2avut4e7pSJ98t50zai2dzNies1OpqTU58BAM1pWI99FxM6thX9
+aDa+Qhz0AxhA9P+3eQCXYTZR7CEE
+=LPl8
+-----END PGP PRIVATE KEY BLOCK-----`;
+
+  const privateKey = await openpgp.readKey({ armoredKey });
+  const publicKey = privateKey.toPublic();
+  const plaintextMessage = await openpgp.createMessage({ text: 'plaintext' });
+  const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, encryptionKeys: publicKey });
+  const armoredSignedMessage = await openpgp.sign({ message: await openpgp.createMessage({ text: 'plaintext' }), signingKeys: privateKey });
+
+  return {
+    armoredKey,
+    privateKey,
+    publicKey,
+    armoredEncryptedMessage,
+    armoredSignedMessage
+  };
+}