suzanne.soy/fork-openpgpjs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add memory benchmark tests for streamed decryption of large files (#1462)

Browse Source 
Also, provide more details about `config.allowUnauthenticatedStream`.
This commit is contained in:
Justin Lovell 2022-06-03 01:29:27 +10:00 committed by GitHub
parent 775dade80f
commit a822dd817b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 145 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/config/config.js
View File

@ -92,7 +92,10 @@ export default {
*/
allowUnauthenticatedMessages: false,
/**
* Allow streaming unauthenticated data before its integrity has been checked.
* Allow streaming unauthenticated data before its integrity has been checked. This would allow the application to
* process large streams while limiting memory usage by releasing the decrypted chunks as soon as possible
* and deferring checking their integrity until the decrypted stream has been read in full.
*
* This setting is **insecure** if the partially decrypted message is processed further or displayed to the user.
* @memberof module:config
* @property {Boolean} allowUnauthenticatedStream

159
test/benchmarks/memory_usage.js
View File

@ -1,6 +1,5 @@
/* eslint-disable no-console */
const assert = require('assert');
const stream = require('@openpgp/web-stream-tools');
const path = require('path');
const { writeFileSync, unlinkSync } = require('fs');
const { fork } = require('child_process');
@ -17,7 +16,6 @@ const benchmark = async function(fn) {
// the code to execute must be written to a file
writeFileSync(tmpFileName, `
const assert = require('assert');
const stream = require('@openpgp/web-stream-tools');
const openpgp = require('../..');
let maxMemoryComsumption;
let activeSampling = false;
@ -111,9 +109,10 @@ class MemoryBenchamrkSuite {
suite.add('empty test (baseline)', () => {});
suite.add('openpgp.encrypt/decrypt (CFB, binary)', async () => {
const ONE_MEGABYTE = 1000000;
const passwords = 'password';
const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const plaintextMessage = await openpgp.createMessage({ binary: new Uint8Array(1000000).fill(1) });
const plaintextMessage = await openpgp.createMessage({ binary: new Uint8Array(ONE_MEGABYTE) });
const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
@ -122,9 +121,10 @@ class MemoryBenchamrkSuite {
});
suite.add('openpgp.encrypt/decrypt (CFB, text)', async () => {
const ONE_MEGABYTE = 1000000;
const passwords = 'password';
const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const plaintextMessage = await openpgp.createMessage({ text: 'a'.repeat(10000000 / 2) }); // two bytes per character
const plaintextMessage = await openpgp.createMessage({ text: 'a'.repeat(ONE_MEGABYTE / 2) }); // two bytes per character
const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
@ -133,9 +133,10 @@ class MemoryBenchamrkSuite {
});
suite.add('openpgp.encrypt/decrypt (AEAD, binary)', async () => {
const ONE_MEGABYTE = 1000000;
const passwords = 'password';
const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const plaintextMessage = await openpgp.createMessage({ binary: new Uint8Array(1000000).fill(1) });
const plaintextMessage = await openpgp.createMessage({ binary: new Uint8Array(ONE_MEGABYTE) });
const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
@ -144,9 +145,10 @@ class MemoryBenchamrkSuite {
});
suite.add('openpgp.encrypt/decrypt (AEAD, text)', async () => {
const ONE_MEGABYTE = 1000000;
const passwords = 'password';
const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const plaintextMessage = await openpgp.createMessage({ text: 'a'.repeat(10000000 / 2) }); // two bytes per character
const plaintextMessage = await openpgp.createMessage({ text: 'a'.repeat(ONE_MEGABYTE / 2) }); // two bytes per character
const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
@ -156,61 +158,182 @@ class MemoryBenchamrkSuite {
// streaming tests
suite.add('openpgp.encrypt/decrypt (CFB, binary, with streaming)', async () => {
await stream.loadStreamsPonyfill();
const ONE_MEGABYTE = 1000000;
function* largeDataGenerator({ chunk, numberOfChunks }) {
for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
yield chunk;
}
}
const passwords = 'password';
const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const plaintextMessage = await openpgp.createMessage({ binary: stream.toStream(new Uint8Array(1000000).fill(1)) });
const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: new Uint8Array(ONE_MEGABYTE), numberOfChunks: 1 }));
const plaintextMessage = await openpgp.createMessage({ binary: inputStream });
assert(plaintextMessage.fromStream);
const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
assert.ok(encryptedMessage.packets[1] instanceof openpgp.SymEncryptedIntegrityProtectedDataPacket);
const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
await stream.readToEnd(decryptedData);
// read out output stream to trigger decryption
await new Promise(resolve => {
decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
decryptedData.on('end', resolve);
});
});
suite.add('openpgp.encrypt/decrypt (CFB, text, with streaming)', async () => {
await stream.loadStreamsPonyfill();
const ONE_MEGABYTE = 1000000;
function* largeDataGenerator({ chunk, numberOfChunks }) {
for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
yield chunk;
}
}
const passwords = 'password';
const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const plaintextMessage = await openpgp.createMessage({ text: stream.toStream('a'.repeat(10000000 / 2)) });
const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: 'a'.repeat(ONE_MEGABYTE / 2), numberOfChunks: 1 }));
const plaintextMessage = await openpgp.createMessage({ text: inputStream });
assert(plaintextMessage.fromStream);
const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
assert.ok(encryptedMessage.packets[1] instanceof openpgp.SymEncryptedIntegrityProtectedDataPacket);
const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
await stream.readToEnd(decryptedData);
// read out output stream to trigger decryption
await new Promise(resolve => {
decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
decryptedData.on('end', resolve);
});
});
suite.add('openpgp.encrypt/decrypt (AEAD, binary, with streaming)', async () => {
await stream.loadStreamsPonyfill();
const ONE_MEGABYTE = 1000000;
function* largeDataGenerator({ chunk, numberOfChunks }) {
for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
yield chunk;
}
}
const passwords = 'password';
const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const plaintextMessage = await openpgp.createMessage({ binary: stream.toStream(new Uint8Array(1000000).fill(1)) });
const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: new Uint8Array(ONE_MEGABYTE), numberOfChunks: 1 }));
const plaintextMessage = await openpgp.createMessage({ binary:inputStream });
assert(plaintextMessage.fromStream);
const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
assert.ok(encryptedMessage.packets[1] instanceof openpgp.AEADEncryptedDataPacket);
await openpgp.decrypt({ message: encryptedMessage, passwords, config });
const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
// read out output stream to trigger decryption
await new Promise(resolve => {
decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
decryptedData.on('end', resolve);
});
});
suite.add('openpgp.encrypt/decrypt (AEAD, text, with streaming)', async () => {
await stream.loadStreamsPonyfill();
const ONE_MEGABYTE = 1000000;
function* largeDataGenerator({ chunk, numberOfChunks }) {
for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
yield chunk;
}
}
const passwords = 'password';
const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const plaintextMessage = await openpgp.createMessage({ text: stream.toStream('a'.repeat(10000000 / 2)) });
const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: 'a'.repeat(ONE_MEGABYTE / 2), numberOfChunks: 1 }));
const plaintextMessage = await openpgp.createMessage({ text: inputStream });
assert(plaintextMessage.fromStream);
const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
assert.ok(encryptedMessage.packets[1] instanceof openpgp.AEADEncryptedDataPacket);
await openpgp.decrypt({ message: encryptedMessage, passwords, config });
const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
// read out output stream to trigger decryption
await new Promise(resolve => {
decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
decryptedData.on('end', resolve);
});
});
suite.add('openpgp.encrypt/decrypt (CFB, text @ 10MB, with streaming)', async () => {
const ONE_MEGABYTE = 1000000;
function* largeDataGenerator({ chunk, numberOfChunks }) {
for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
yield chunk;
}
}
const passwords = 'password';
const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: 'a'.repeat(ONE_MEGABYTE / 2), numberOfChunks: 20 }));
const plaintextMessage = await openpgp.createMessage({ text: inputStream });
assert(plaintextMessage.fromStream);
const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
assert.ok(encryptedMessage.packets[1] instanceof openpgp.SymEncryptedIntegrityProtectedDataPacket);
const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
// read out output stream to trigger decryption
await new Promise(resolve => {
decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
decryptedData.on('end', resolve);
});
});
suite.add('openpgp.encrypt/decrypt (CFB, text @ 10MB, with unauthenticated streaming)', async () => {
const ONE_MEGABYTE = 1000000;
function* largeDataGenerator({ chunk, numberOfChunks }) {
for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
yield chunk;
}
}
const passwords = 'password';
const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: 'a'.repeat(ONE_MEGABYTE / 2), numberOfChunks: 20 }));
const plaintextMessage = await openpgp.createMessage({ text: inputStream });
assert(plaintextMessage.fromStream);
const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
assert.ok(encryptedMessage.packets[1] instanceof openpgp.SymEncryptedIntegrityProtectedDataPacket);
const { data: decryptedData } = await openpgp.decrypt({
message: encryptedMessage,
passwords,
config: { ...config, allowUnauthenticatedStream: true }
});
// read out output stream to trigger decryption
await new Promise(resolve => {
decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
decryptedData.on('end', resolve);
});
});
suite.add('openpgp.encrypt/decrypt (AEAD, text @ 10MB, with streaming)', async () => {
const ONE_MEGABYTE = 1000000;
function* largeDataGenerator({ chunk, numberOfChunks }) {
for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
yield chunk;
}
}
const passwords = 'password';
const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: 'a'.repeat(ONE_MEGABYTE / 2), numberOfChunks: 20 }));
const plaintextMessage = await openpgp.createMessage({ text: inputStream });
assert(plaintextMessage.fromStream);
const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
assert.ok(encryptedMessage.packets[1] instanceof openpgp.AEADEncryptedDataPacket);
const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
// read out output stream to trigger decryption
await new Promise(resolve => {
decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
decryptedData.on('end', resolve);
});
});
const stats = await suite.run();