From 9c1c28bc5984b17395e0c7083086b8b918f61a2e Mon Sep 17 00:00:00 2001 From: Daniel Huigens Date: Fri, 1 Jun 2018 14:43:34 +0200 Subject: [PATCH] Add option to read unauthenticated data from stream --- src/message.js | 32 ++- src/packet/packet.js | 10 +- src/packet/packetlist.js | 38 ++-- .../sym_encrypted_integrity_protected.js | 22 +- src/stream.js | 46 +++- src/util.js | 2 + test/general/streaming.js | 206 +++++++++++++++++- 7 files changed, 299 insertions(+), 57 deletions(-) diff --git a/src/message.js b/src/message.js index c16e373a..17610eda 100644 --- a/src/message.js +++ b/src/message.js @@ -545,24 +545,22 @@ Message.prototype.verify = async function(keys, date=new Date()) { if (msg.packets.stream) { let onePassSigList = msg.packets.filterByTag(enums.packet.onePassSignature); onePassSigList = Array.from(onePassSigList).reverse(); - if (onePassSigList.length) { - onePassSigList.forEach(onePassSig => { - onePassSig.signatureData = stream.fromAsync(() => new Promise(resolve => { - onePassSig.signatureDataResolve = resolve; - })); - onePassSig.hashed = onePassSig.hash(literalDataList[0]); - }); - const reader = stream.getReader(msg.packets.stream); - for (let i = 0; ; i++) { - const { done, value } = await reader.read(); - if (done) { - break; - } - onePassSigList[i].signatureDataResolve(value.signatureData); - value.hashed = onePassSigList[i].hashed; - value.hashedData = onePassSigList[i].hashedData; - msg.packets.push(value); + onePassSigList.forEach(onePassSig => { + onePassSig.signatureData = stream.fromAsync(() => new Promise(resolve => { + onePassSig.signatureDataResolve = resolve; + })); + onePassSig.hashed = onePassSig.hash(literalDataList[0]); + }); + const reader = stream.getReader(msg.packets.stream); + for (let i = 0; ; i++) { + const { done, value } = await reader.read(); + if (done) { + break; } + onePassSigList[i].signatureDataResolve(value.signatureData); + value.hashed = onePassSigList[i].hashed; + value.hashedData = onePassSigList[i].hashedData; + msg.packets.push(value); } } const signatureList = msg.packets.filterByTag(enums.packet.signature); diff --git a/src/packet/packet.js b/src/packet/packet.js index a8ff1a43..1ab4da36 100644 --- a/src/packet/packet.js +++ b/src/packet/packet.js @@ -142,9 +142,6 @@ export default { let controller; try { const peekedBytes = await reader.peekBytes(2); - if (!peekedBytes || !peekedBytes.length) { - return false; - } // some sanity checks if (!peekedBytes || peekedBytes.length < 2 || (peekedBytes[0] & 0x80) === 0) { throw new Error("Error during parsing. This message / key probably does not conform to a valid OpenPGP format."); @@ -255,9 +252,13 @@ export default { packet = await reader.readBytes(packet_length); await callback({ tag, packet }); } - } else if (controller) { + } + const { done, value } = await reader.read(); + if (!done) reader.unshift(value); + if (controller) { controller.close(); } + return !done && value && value.length; } catch(e) { if (controller) { controller.error(e); @@ -267,6 +268,5 @@ export default { } finally { reader.releaseLock(); } - return true; } }; diff --git a/src/packet/packetlist.js b/src/packet/packetlist.js index 825c7db7..1fea7db7 100644 --- a/src/packet/packetlist.js +++ b/src/packet/packetlist.js @@ -38,25 +38,29 @@ function List() { List.prototype.read = async function (bytes) { this.stream = new ReadableStream({ pull: async controller => { - if (!await packetParser.read(bytes, async parsed => { - try { - const tag = enums.read(enums.packet, parsed.tag); - const packet = packets.newPacketFromTag(tag); - packet.packets = new List(); - packet.fromStream = util.isStream(parsed.packet); - await packet.read(parsed.packet); - controller.enqueue(packet); - } catch (e) { - if (!config.tolerant || - parsed.tag === enums.packet.symmetricallyEncrypted || - parsed.tag === enums.packet.literal || - parsed.tag === enums.packet.compressed) { - controller.error(e); + try { + if (!await packetParser.read(bytes, async parsed => { + try { + const tag = enums.read(enums.packet, parsed.tag); + const packet = packets.newPacketFromTag(tag); + packet.packets = new List(); + packet.fromStream = util.isStream(parsed.packet); + await packet.read(parsed.packet); + controller.enqueue(packet); + } catch (e) { + if (!config.tolerant || + parsed.tag === enums.packet.symmetricallyEncrypted || + parsed.tag === enums.packet.literal || + parsed.tag === enums.packet.compressed) { + controller.error(e); + } + util.print_debug_error(e); } - util.print_debug_error(e); + })) { + controller.close(); } - })) { - controller.close(); + } catch(e) { + controller.error(e); } } }); diff --git a/src/packet/sym_encrypted_integrity_protected.js b/src/packet/sym_encrypted_integrity_protected.js index 5ec88ba1..12ebbea8 100644 --- a/src/packet/sym_encrypted_integrity_protected.js +++ b/src/packet/sym_encrypted_integrity_protected.js @@ -17,6 +17,7 @@ /** * @requires asmcrypto.js + * @requires config * @requires crypto * @requires enums * @requires stream @@ -25,6 +26,7 @@ import { AES_CFB_Decrypt, AES_CFB_Encrypt } from 'asmcrypto.js/src/aes/cfb/exports'; +import config from '../config'; import crypto from '../crypto'; import enums from '../enums'; import stream from '../stream'; @@ -131,15 +133,21 @@ SymEncryptedIntegrityProtected.prototype.decrypt = async function (sessionKeyAlg const prefix = crypto.cfb.mdc(sessionKeyAlgorithm, key, encryptedPrefix); const bytes = stream.slice(stream.clone(decrypted), 0, -20); const tohash = util.concat([prefix, stream.clone(bytes)]); - this.hash = util.Uint8Array_to_str(await stream.readToEnd(crypto.hash.sha1(tohash))); - const mdc = util.Uint8Array_to_str(await stream.readToEnd(stream.slice(decrypted, -20))); - - if (this.hash !== mdc) { - throw new Error('Modification detected.'); + const verifyHash = Promise.all([ + stream.readToEnd(crypto.hash.sha1(tohash)), + stream.readToEnd(stream.slice(decrypted, -20)) + ]).then(([hash, mdc]) => { + if (!util.equalsUint8Array(hash, mdc)) { + throw new Error('Modification detected.'); + } + }); + let packetbytes = stream.slice(bytes, 0, -2); + if (!util.isStream(encrypted) || !config.unsafe_stream) { + await verifyHash; } else { - await this.packets.read(stream.slice(bytes, 0, -2)); + packetbytes = stream.concat([packetbytes, stream.fromAsync(() => verifyHash)]); } - + await this.packets.read(packetbytes); return true; }; diff --git a/src/stream.js b/src/stream.js index f225d080..74f00559 100644 --- a/src/stream.js +++ b/src/stream.js @@ -11,13 +11,17 @@ function concat(arrays) { let current = 0; return new ReadableStream({ async pull(controller) { - const { done, value } = await readers[current].read(); - if (!done) { - controller.enqueue(value); - } else if (++current === arrays.length) { - controller.close(); - } else { - await this.pull(controller); // ??? Chrome bug? + try { + const { done, value } = await readers[current].read(); + if (!done) { + controller.enqueue(value); + } else if (++current === arrays.length) { + controller.close(); + } else { + await this.pull(controller); // ??? Chrome bug? + } + } catch(e) { + controller.error(e); } } }); @@ -97,7 +101,27 @@ function slice(input, begin=0, end=Infinity) { } }); } + if (begin < 0 && (end < 0 || end === Infinity)) { + let lastBytes = []; + return transform(input, value => { + if (value.length >= -begin) lastBytes = [value]; + else lastBytes.push(value); + }, () => slice(util.concat(lastBytes), begin, end)); + } + if (begin === 0 && end < 0) { + let lastBytes; + return transform(input, value => { + const returnValue = lastBytes ? util.concat([lastBytes, value]) : value; + if (returnValue.length >= -end) { + lastBytes = slice(returnValue, end); + return slice(returnValue, begin, end); + } else { + lastBytes = returnValue; + } + }); + } // TODO: Don't read entire stream into memory here. + util.print_debug_error(`stream.slice(input, ${begin}, ${end}) not implemented efficiently.`); return fromAsync(async () => slice(await readToEnd(input), begin, end)); } if (input.externalBuffer) { @@ -125,8 +149,12 @@ async function cancel(input) { function fromAsync(fn) { return new ReadableStream({ pull: async controller => { - controller.enqueue(await fn()); - controller.close(); + try { + controller.enqueue(await fn()); + controller.close(); + } catch(e) { + controller.error(e); + } } }); } diff --git a/src/util.js b/src/util.js index c8db4648..2507627c 100644 --- a/src/util.js +++ b/src/util.js @@ -347,6 +347,8 @@ export default { * @returns {Uint8array} Concatenated array */ concatUint8Array: function (arrays) { + if (arrays.length === 1) return arrays[0]; + let totalLength = 0; for (let i = 0; i < arrays.length; i++) { if (!util.isUint8Array(arrays[i])) { diff --git a/test/general/streaming.js b/test/general/streaming.js index 2a20065c..47e17a18 100644 --- a/test/general/streaming.js +++ b/test/general/streaming.js @@ -8,6 +8,73 @@ const { expect } = chai; const { util } = openpgp; +const pub_key = + ['-----BEGIN PGP PUBLIC KEY BLOCK-----', + 'Version: GnuPG v2.0.19 (GNU/Linux)', + '', + 'mI0EUmEvTgEEANyWtQQMOybQ9JltDqmaX0WnNPJeLILIM36sw6zL0nfTQ5zXSS3+', + 'fIF6P29lJFxpblWk02PSID5zX/DYU9/zjM2xPO8Oa4xo0cVTOTLj++Ri5mtr//f5', + 'GLsIXxFrBJhD/ghFsL3Op0GXOeLJ9A5bsOn8th7x6JucNKuaRB6bQbSPABEBAAG0', + 'JFRlc3QgTWNUZXN0aW5ndG9uIDx0ZXN0QGV4YW1wbGUuY29tPoi5BBMBAgAjBQJS', + 'YS9OAhsvBwsJCAcDAgEGFQgCCQoLBBYCAwECHgECF4AACgkQSmNhOk1uQJQwDAP6', + 'AgrTyqkRlJVqz2pb46TfbDM2TDF7o9CBnBzIGoxBhlRwpqALz7z2kxBDmwpQa+ki', + 'Bq3jZN/UosY9y8bhwMAlnrDY9jP1gdCo+H0sD48CdXybblNwaYpwqC8VSpDdTndf', + '9j2wE/weihGp/DAdy/2kyBCaiOY1sjhUfJ1GogF49rC4jQRSYS9OAQQA6R/PtBFa', + 'JaT4jq10yqASk4sqwVMsc6HcifM5lSdxzExFP74naUMMyEsKHP53QxTF0Grqusag', + 'Qg/ZtgT0CN1HUM152y7ACOdp1giKjpMzOTQClqCoclyvWOFB+L/SwGEIJf7LSCEr', + 'woBuJifJc8xAVr0XX0JthoW+uP91eTQ3XpsAEQEAAYkBPQQYAQIACQUCUmEvTgIb', + 'LgCoCRBKY2E6TW5AlJ0gBBkBAgAGBQJSYS9OAAoJEOCE90RsICyXuqIEANmmiRCA', + 'SF7YK7PvFkieJNwzeK0V3F2lGX+uu6Y3Q/Zxdtwc4xR+me/CSBmsURyXTO29OWhP', + 'GLszPH9zSJU9BdDi6v0yNprmFPX/1Ng0Abn/sCkwetvjxC1YIvTLFwtUL/7v6NS2', + 'bZpsUxRTg9+cSrMWWSNjiY9qUKajm1tuzPDZXAUEAMNmAN3xXN/Kjyvj2OK2ck0X', + 'W748sl/tc3qiKPMJ+0AkMF7Pjhmh9nxqE9+QCEl7qinFqqBLjuzgUhBU4QlwX1GD', + 'AtNTq6ihLMD5v1d82ZC7tNatdlDMGWnIdvEMCv2GZcuIqDQ9rXWs49e7tq1NncLY', + 'hz3tYjKhoFTKEIq3y3Pp', + '=h/aX', + '-----END PGP PUBLIC KEY BLOCK-----'].join('\n'); + +const priv_key = + ['-----BEGIN PGP PRIVATE KEY BLOCK-----', + 'Version: GnuPG v2.0.19 (GNU/Linux)', + '', + 'lQH+BFJhL04BBADclrUEDDsm0PSZbQ6pml9FpzTyXiyCyDN+rMOsy9J300Oc10kt', + '/nyBej9vZSRcaW5VpNNj0iA+c1/w2FPf84zNsTzvDmuMaNHFUzky4/vkYuZra//3', + '+Ri7CF8RawSYQ/4IRbC9zqdBlzniyfQOW7Dp/LYe8eibnDSrmkQem0G0jwARAQAB', + '/gMDAu7L//czBpE40p1ZqO8K3k7UejemjsQqc7kOqnlDYd1Z6/3NEA/UM30Siipr', + 'KjdIFY5+hp0hcs6EiiNq0PDfm/W2j+7HfrZ5kpeQVxDek4irezYZrl7JS2xezaLv', + 'k0Fv/6fxasnFtjOM6Qbstu67s5Gpl9y06ZxbP3VpT62+Xeibn/swWrfiJjuGEEhM', + 'bgnsMpHtzAz/L8y6KSzViG/05hBaqrvk3/GeEA6nE+o0+0a6r0LYLTemmq6FbaA1', + 'PHo+x7k7oFcBFUUeSzgx78GckuPwqr2mNfeF+IuSRnrlpZl3kcbHASPAOfEkyMXS', + 'sWGE7grCAjbyQyM3OEXTSyqnehvGS/1RdB6kDDxGwgE/QFbwNyEh6K4eaaAThW2j', + 'IEEI0WEnRkPi9fXyxhFsCLSI1XhqTaq7iDNqJTxE+AX2b9ZuZXAxI3Tc/7++vEyL', + '3p18N/MB2kt1Wb1azmXWL2EKlT1BZ5yDaJuBQ8BhphM3tCRUZXN0IE1jVGVzdGlu', + 'Z3RvbiA8dGVzdEBleGFtcGxlLmNvbT6IuQQTAQIAIwUCUmEvTgIbLwcLCQgHAwIB', + 'BhUIAgkKCwQWAgMBAh4BAheAAAoJEEpjYTpNbkCUMAwD+gIK08qpEZSVas9qW+Ok', + '32wzNkwxe6PQgZwcyBqMQYZUcKagC8+89pMQQ5sKUGvpIgat42Tf1KLGPcvG4cDA', + 'JZ6w2PYz9YHQqPh9LA+PAnV8m25TcGmKcKgvFUqQ3U53X/Y9sBP8HooRqfwwHcv9', + 'pMgQmojmNbI4VHydRqIBePawnQH+BFJhL04BBADpH8+0EVolpPiOrXTKoBKTiyrB', + 'UyxzodyJ8zmVJ3HMTEU/vidpQwzISwoc/ndDFMXQauq6xqBCD9m2BPQI3UdQzXnb', + 'LsAI52nWCIqOkzM5NAKWoKhyXK9Y4UH4v9LAYQgl/stIISvCgG4mJ8lzzEBWvRdf', + 'Qm2Ghb64/3V5NDdemwARAQAB/gMDAu7L//czBpE40iPcpLzL7GwBbWFhSWgSLy53', + 'Md99Kxw3cApWCok2E8R9/4VS0490xKZIa5y2I/K8thVhqk96Z8Kbt7MRMC1WLHgC', + 'qJvkeQCI6PrFM0PUIPLHAQtDJYKtaLXxYuexcAdKzZj3FHdtLNWCooK6n3vJlL1c', + 'WjZcHJ1PH7USlj1jup4XfxsbziuysRUSyXkjn92GZLm+64vCIiwhqAYoizF2NHHG', + 'hRTN4gQzxrxgkeVchl+ag7DkQUDANIIVI+A63JeLJgWJiH1fbYlwESByHW+zBFNt', + 'qStjfIOhjrfNIc3RvsggbDdWQLcbxmLZj4sB0ydPSgRKoaUdRHJY0S4vp9ouKOtl', + '2au/P1BP3bhD0fDXl91oeheYth+MSmsJFDg/vZJzCJhFaQ9dp+2EnjN5auNCNbaI', + 'beFJRHFf9cha8p3hh+AK54NRCT++B2MXYf+TPwqX88jYMBv8kk8vYUgo8128r1zQ', + 'EzjviQE9BBgBAgAJBQJSYS9OAhsuAKgJEEpjYTpNbkCUnSAEGQECAAYFAlJhL04A', + 'CgkQ4IT3RGwgLJe6ogQA2aaJEIBIXtgrs+8WSJ4k3DN4rRXcXaUZf667pjdD9nF2', + '3BzjFH6Z78JIGaxRHJdM7b05aE8YuzM8f3NIlT0F0OLq/TI2muYU9f/U2DQBuf+w', + 'KTB62+PELVgi9MsXC1Qv/u/o1LZtmmxTFFOD35xKsxZZI2OJj2pQpqObW27M8Nlc', + 'BQQAw2YA3fFc38qPK+PY4rZyTRdbvjyyX+1zeqIo8wn7QCQwXs+OGaH2fGoT35AI', + 'SXuqKcWqoEuO7OBSEFThCXBfUYMC01OrqKEswPm/V3zZkLu01q12UMwZach28QwK', + '/YZly4ioND2tdazj17u2rU2dwtiHPe1iMqGgVMoQirfLc+k=', + '=lw5e', + '-----END PGP PRIVATE KEY BLOCK-----'].join('\n'); + +const passphrase = 'hello world'; + describe('Streaming', function() { it('Encrypt small message', async function() { const data = new ReadableStream({ @@ -48,7 +115,7 @@ describe('Streaming', function() { data, passwords: ['test'], }); - await openpgp.stream.getReader(openpgp.stream.clone(encrypted.data)).readBytes(1000); + expect(await openpgp.stream.getReader(openpgp.stream.clone(encrypted.data)).readBytes(1024)).to.match(/^-----BEGIN PGP MESSAGE-----\r\nVersion: OpenPGP.js VERSION\r\nComment: https:\/\/openpgpjs.org\r\n\r\n/); if (i > 10) throw new Error('Data did not arrive early.'); const msgAsciiArmored = await openpgp.stream.readToEnd(encrypted.data); const message = await openpgp.message.readArmored(msgAsciiArmored); @@ -87,9 +154,144 @@ describe('Streaming', function() { format: 'binary' }); expect(util.isStream(decrypted.data)).to.be.true; + expect(await openpgp.stream.getReader(openpgp.stream.clone(decrypted.data)).readBytes(1024)).to.deep.equal(plaintext[0]); + if (i <= 10) throw new Error('Data arrived early.'); expect(await openpgp.stream.readToEnd(decrypted.data)).to.deep.equal(util.concatUint8Array(plaintext)); }); + it('Encrypt and decrypt larger message roundtrip (unsafe_stream=true)', async function() { + let unsafe_streamValue = openpgp.config.unsafe_stream; + openpgp.config.unsafe_stream = true; + try { + let plaintext = []; + let i = 0; + const data = new ReadableStream({ + async pull(controller) { + await new Promise(setTimeout); + if (i++ < 10) { + let randomBytes = await openpgp.crypto.random.getRandomBytes(1024); + controller.enqueue(randomBytes); + plaintext.push(randomBytes); + } else { + controller.close(); + } + } + }); + const encrypted = await openpgp.encrypt({ + data, + passwords: ['test'], + }); + + const msgAsciiArmored = encrypted.data; + const message = await openpgp.message.readArmored(msgAsciiArmored); + const decrypted = await openpgp.decrypt({ + passwords: ['test'], + message, + format: 'binary' + }); + expect(util.isStream(decrypted.data)).to.be.true; + expect(await openpgp.stream.getReader(openpgp.stream.clone(decrypted.data)).readBytes(1024)).to.deep.equal(plaintext[0]); + if (i > 10) throw new Error('Data did not arrive early.'); + expect(await openpgp.stream.readToEnd(decrypted.data)).to.deep.equal(util.concatUint8Array(plaintext)); + expect(await decrypted.signatures).to.exist.and.have.length(0); + } finally { + openpgp.config.unsafe_stream = unsafe_streamValue; + } + }); + + it('Detect MDC modifications (unsafe_stream=true)', async function() { + let unsafe_streamValue = openpgp.config.unsafe_stream; + openpgp.config.unsafe_stream = true; + try { + let plaintext = []; + let i = 0; + const data = new ReadableStream({ + async pull(controller) { + await new Promise(setTimeout); + if (i++ < 10) { + let randomBytes = await openpgp.crypto.random.getRandomBytes(1024); + controller.enqueue(randomBytes); + plaintext.push(randomBytes); + } else { + controller.close(); + } + } + }); + const encrypted = await openpgp.encrypt({ + data, + passwords: ['test'], + }); + + const msgAsciiArmored = encrypted.data; + const message = await openpgp.message.readArmored(openpgp.stream.transform(msgAsciiArmored, value => { + if (value === '\n=' || value.length === 4) return; // Remove checksum + if (value.length > 1000) return value.slice(0, 499) + 'a' + value.slice(500); + return value; + })); + const decrypted = await openpgp.decrypt({ + passwords: ['test'], + message, + format: 'binary' + }); + expect(util.isStream(decrypted.data)).to.be.true; + expect(await openpgp.stream.getReader(openpgp.stream.clone(decrypted.data)).readBytes(1024)).not.to.deep.equal(plaintext[0]); + if (i > 10) throw new Error('Data did not arrive early.'); + await expect(openpgp.stream.readToEnd(decrypted.data)).to.be.rejectedWith('Modification detected.'); + await decrypted.signatures; + } finally { + openpgp.config.unsafe_stream = unsafe_streamValue; + } + }); + + it('Detect armor checksum error (unsafe_stream=true)', async function() { + let unsafe_streamValue = openpgp.config.unsafe_stream; + openpgp.config.unsafe_stream = true; + try { + const pubKey = (await openpgp.key.readArmored(pub_key)).keys[0]; + const privKey = (await openpgp.key.readArmored(priv_key)).keys[0]; + await privKey.decrypt(passphrase); + + let plaintext = []; + let i = 0; + const data = new ReadableStream({ + async pull(controller) { + await new Promise(resolve => setTimeout(resolve, 100)); + if (i++ < 10) { + let randomBytes = await openpgp.crypto.random.getRandomBytes(1024); + controller.enqueue(randomBytes); + plaintext.push(randomBytes); + } else { + controller.close(); + } + } + }); + const encrypted = await openpgp.encrypt({ + data, + publicKeys: pubKey, + privateKeys: privKey + }); + + const msgAsciiArmored = encrypted.data; + const message = await openpgp.message.readArmored(openpgp.stream.transform(msgAsciiArmored, value => { + if (value.length > 1000) return value.slice(0, 499) + 'a' + value.slice(500); + return value; + })); + const decrypted = await openpgp.decrypt({ + publicKeys: pubKey, + privateKeys: privKey, + message, + format: 'binary' + }); + expect(util.isStream(decrypted.data)).to.be.true; + expect(await openpgp.stream.getReader(openpgp.stream.clone(decrypted.data)).readBytes(10)).not.to.deep.equal(plaintext[0]); + if (i > 10) throw new Error('Data did not arrive early.'); + await openpgp.stream.readToEnd(decrypted.data); + expect(decrypted.signatures).to.be.rejectedWith('Ascii armor integrity check on message failed'); + } finally { + openpgp.config.unsafe_stream = unsafe_streamValue; + } + }); + it('Encrypt and decrypt larger message roundtrip (draft04)', async function() { let aead_protectValue = openpgp.config.aead_protect; let aead_chunk_size_byteValue = openpgp.config.aead_chunk_size_byte; @@ -123,7 +325,7 @@ describe('Streaming', function() { format: 'binary' }); expect(util.isStream(decrypted.data)).to.be.true; - await openpgp.stream.getReader(openpgp.stream.clone(decrypted.data)).readBytes(1000); + expect(await openpgp.stream.getReader(openpgp.stream.clone(decrypted.data)).readBytes(1024)).to.deep.equal(plaintext[0]); if (i > 10) throw new Error('Data did not arrive early.'); expect(await openpgp.stream.readToEnd(decrypted.data)).to.deep.equal(util.concatUint8Array(plaintext)); } finally {