diff --git a/src/encoding/armor.js b/src/encoding/armor.js index a14b29c3..bb2409a3 100644 --- a/src/encoding/armor.js +++ b/src/encoding/armor.js @@ -119,7 +119,7 @@ function addheader(customComment) { /** * Calculates a checksum over the given data and returns it base64 encoded * @param {String} data Data to create a CRC-24 checksum for - * @returns {Uint8Array} Base64 encoded checksum + * @returns {String} Base64 encoded checksum */ function getCheckSum(data) { const crc = createcrc24(data); @@ -201,9 +201,6 @@ function verifyHeaders(headers) { * @static */ function dearmor(input) { - if (util.isString(input)) { - input = util.str_to_Uint8Array(util.encode_utf8(input)); - } return new Promise(async (resolve, reject) => { try { const reSplit = /^-----[^-]+-----$/; @@ -227,7 +224,7 @@ function dearmor(input) { const checksumVerified = getCheckSum(dataClone); data = stream.getReader(data).substream(); // Convert to Stream data = stream.transform(data, value => value, async () => { - const checksumVerifiedString = util.Uint8Array_to_str(await stream.readToEnd(checksumVerified)); + const checksumVerifiedString = await stream.readToEnd(checksumVerified); if (checksum !== checksumVerifiedString && (checksum || config.checksum_required)) { throw new Error("Ascii armor integrity check on message failed: '" + checksum + "' should be '" + checksumVerifiedString + "'"); @@ -236,7 +233,6 @@ function dearmor(input) { while (true) { let line = await reader.readLine(); if (!line) break; - line = util.decode_utf8(util.Uint8Array_to_str(line)); if (lineIndex++ === 0) { // trim string line = line.trim(); @@ -272,7 +268,7 @@ function dearmor(input) { } else { if (!reSplit.test(line)) { if (line[0] !== '=') { - controller.enqueue(util.str_to_Uint8Array(line)); + controller.enqueue(line); } else { checksum = line.substr(1); } @@ -365,7 +361,7 @@ function armor(messagetype, body, partindex, parttotal, customComment) { break; } - return util.concatUint8Array(result.map(part => (util.isString(part) ? util.str_to_Uint8Array(part) : part))); + return stream.concat(result); } export default { diff --git a/src/encoding/base64.js b/src/encoding/base64.js index 135fd757..78a54663 100644 --- a/src/encoding/base64.js +++ b/src/encoding/base64.js @@ -13,15 +13,13 @@ /** * @requires stream - * @requires util * @module encoding/base64 */ import stream from '../stream'; -import util from '../util'; -const b64s = util.str_to_Uint8Array('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'); // Standard radix-64 -const b64u = util.str_to_Uint8Array('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'); // URL-safe radix-64 +const b64s = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; // Standard radix-64 +const b64u = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'; // URL-safe radix-64 /** * Convert binary array to radix-64 @@ -45,22 +43,22 @@ function s2r(t, u = false) { for (let n = 0; n < tl; n++) { c = value[n]; if (s === 0) { - r.push(b64[(c >> 2) & 63]); + r.push(b64.charAt((c >> 2) & 63)); a = (c & 3) << 4; } else if (s === 1) { - r.push(b64[a | ((c >> 4) & 15)]); + r.push(b64.charAt(a | ((c >> 4) & 15))); a = (c & 15) << 2; } else if (s === 2) { - r.push(b64[a | ((c >> 6) & 3)]); + r.push(b64.charAt(a | ((c >> 6) & 3))); l += 1; if ((l % 60) === 0 && !u) { - r.push(10); // "\n" + r.push("\n"); } - r.push(b64[c & 63]); + r.push(b64.charAt(c & 63)); } l += 1; if ((l % 60) === 0 && !u) { - r.push(10); // "\n" + r.push("\n"); } s += 1; @@ -68,27 +66,27 @@ function s2r(t, u = false) { s = 0; } } - return new Uint8Array(r); + return r.join(''); }, () => { const r = []; if (s > 0) { - r.push(b64[a]); + r.push(b64.charAt(a)); l += 1; if ((l % 60) === 0 && !u) { - r.push(10); // "\n" + r.push("\n"); } if (!u) { - r.push(61); // "=" + r.push('='); l += 1; } } if (s === 1 && !u) { if ((l % 60) === 0 && !u) { - r.push(10); // "\n" + r.push("\n"); } - r.push(61); // "=" + r.push('='); } - return new Uint8Array(r); + return r.join(''); }); } @@ -111,7 +109,7 @@ function r2s(t, u) { const r = []; const tl = value.length; for (let n = 0; n < tl; n++) { - c = b64.indexOf(value[n]); + c = b64.indexOf(value.charAt(n)); if (c >= 0) { if (s) { r.push(a | ((c >> (6 - s)) & 255)); diff --git a/src/openpgp.js b/src/openpgp.js index ac1e9946..b7469e78 100644 --- a/src/openpgp.js +++ b/src/openpgp.js @@ -323,12 +323,15 @@ export function encrypt({ data, dataType, publicKeys, privateKeys, passwords, se message = message.compress(compression); return message.encrypt(publicKeys, passwords, sessionKey, wildcard, date, toUserId); - }).then(encrypted => { + }).then(async encrypted => { + let message = encrypted.message; if (armor) { - result.data = encrypted.message.armor(); - } else { - result.message = encrypted.message; + message = message.armor(); } + if (util.isStream(message) && !util.isStream(data)) { + message = await stream.readToEnd(message); + } + result[armor ? 'data' : 'message'] = message; if (returnSessionKey) { result.sessionKey = encrypted.sessionKey; } @@ -597,17 +600,14 @@ async function parseMessage(message, format, asStream) { let data; if (format === 'binary') { data = message.getLiteralData(); - if (!asStream && util.isStream(data)) { - data = await stream.readToEnd(data); - } } else if (format === 'utf8') { data = message.getText(); - if (!asStream && util.isStream(data)) { - data = await stream.readToEnd(data, chunks => chunks.join('')); - } } else { throw new Error('Invalid format'); } + if (!asStream && util.isStream(data)) { + data = await stream.readToEnd(data); + } const filename = message.getFilename(); return { data, filename }; } diff --git a/src/packet/literal.js b/src/packet/literal.js index 6e2c56a6..d4af1f38 100644 --- a/src/packet/literal.js +++ b/src/packet/literal.js @@ -75,7 +75,7 @@ Literal.prototype.getText = function() { // decode UTF8 and normalize EOL to \n const normalized = normalize(text); // if last two bytes are \r\n or an UTF8 sequence, return them immediately - if (text.length >= 2 && normalize(text.slice(-2)).length === 1) { + if (text.length >= 2 && text.slice(-2) !== normalized.slice(-2)) { lastChar = ''; return normalized; } diff --git a/src/packet/sym_encrypted_aead_protected.js b/src/packet/sym_encrypted_aead_protected.js index 34f2a2c4..821a2a86 100644 --- a/src/packet/sym_encrypted_aead_protected.js +++ b/src/packet/sym_encrypted_aead_protected.js @@ -19,12 +19,14 @@ * @requires config * @requires crypto * @requires enums + * @requires stream * @requires util */ import config from '../config'; import crypto from '../crypto'; import enums from '../enums'; +import stream from '../stream'; import util from '../util'; const VERSION = 1; // A one-octet version number of the data packet. @@ -55,23 +57,21 @@ export default SymEncryptedAEADProtected; /** * Parse an encrypted payload of bytes in the order: version, IV, ciphertext (see specification) */ -SymEncryptedAEADProtected.prototype.read = function (bytes) { - let offset = 0; - if (bytes[offset] !== VERSION) { // The only currently defined value is 1. +SymEncryptedAEADProtected.prototype.read = async function (bytes) { + const reader = stream.getReader(bytes); + if (await reader.readByte() !== VERSION) { // The only currently defined value is 1. throw new Error('Invalid packet version.'); } - offset++; if (config.aead_protect_version === 4) { - this.cipherAlgo = bytes[offset++]; - this.aeadAlgo = bytes[offset++]; - this.chunkSizeByte = bytes[offset++]; + this.cipherAlgo = await reader.readByte(); + this.aeadAlgo = await reader.readByte(); + this.chunkSizeByte = await reader.readByte(); } else { this.aeadAlgo = enums.aead.experimental_gcm; } const mode = crypto[enums.read(enums.aead, this.aeadAlgo)]; - this.iv = bytes.subarray(offset, mode.ivLength + offset); - offset += mode.ivLength; - this.encrypted = bytes.subarray(offset, bytes.length); + this.iv = await reader.readBytes(mode.ivLength); + this.encrypted = reader.substream(); }; /** @@ -93,15 +93,10 @@ SymEncryptedAEADProtected.prototype.write = function () { * @async */ SymEncryptedAEADProtected.prototype.decrypt = async function (sessionKeyAlgorithm, key) { - const mode = crypto[enums.read(enums.aead, this.aeadAlgo)]; - if (config.aead_protect_version === 4) { - const data = this.encrypted.subarray(0, -mode.tagLength); - const authTag = this.encrypted.subarray(-mode.tagLength); - await this.packets.read(await this.crypt('decrypt', key, data, authTag)); - } else { + if (config.aead_protect_version !== 4) { this.cipherAlgo = enums.write(enums.symmetric, sessionKeyAlgorithm); - await this.packets.read(await this.crypt('decrypt', key, this.encrypted)); } + await this.packets.read(await this.crypt('decrypt', key, this.encrypted)); return true; }; @@ -119,7 +114,7 @@ SymEncryptedAEADProtected.prototype.encrypt = async function (sessionKeyAlgorith this.iv = await crypto.random.getRandomBytes(mode.ivLength); // generate new random IV this.chunkSizeByte = config.aead_chunk_size_byte; const data = this.packets.write(); - this.encrypted = await this.crypt('encrypt', key, data, data.subarray(0, 0)); + this.encrypted = await this.crypt('encrypt', key, data); }; /** @@ -127,11 +122,10 @@ SymEncryptedAEADProtected.prototype.encrypt = async function (sessionKeyAlgorith * @param {encrypt|decrypt} fn Whether to encrypt or decrypt * @param {Uint8Array} key The session key used to en/decrypt the payload * @param {Uint8Array} data The data to en/decrypt - * @param {Uint8Array} finalChunk For encryption: empty final chunk; for decryption: final authentication tag * @returns {Promise} * @async */ -SymEncryptedAEADProtected.prototype.crypt = async function (fn, key, data, finalChunk) { +SymEncryptedAEADProtected.prototype.crypt = async function (fn, key, data) { const cipher = enums.read(enums.symmetric, this.cipherAlgo); const mode = crypto[enums.read(enums.aead, this.aeadAlgo)]; const modeInstance = await mode(cipher, key); @@ -144,24 +138,48 @@ SymEncryptedAEADProtected.prototype.crypt = async function (fn, key, data, final const adataView = new DataView(adataBuffer); const chunkIndexArray = new Uint8Array(adataBuffer, 5, 8); adataArray.set([0xC0 | this.tag, this.version, this.cipherAlgo, this.aeadAlgo, this.chunkSizeByte], 0); - adataView.setInt32(13 + 4, data.length - tagLengthIfDecrypting * Math.ceil(data.length / chunkSize)); // Should be setInt64(13, ...) - const cryptedPromises = []; - for (let chunkIndex = 0; chunkIndex === 0 || data.length;) { - cryptedPromises.push( - modeInstance[fn](data.subarray(0, chunkSize), mode.getNonce(this.iv, chunkIndexArray), adataArray) - ); - // We take a chunk of data, en/decrypt it, and shift `data` to the - // next chunk. - data = data.subarray(chunkSize); - adataView.setInt32(5 + 4, ++chunkIndex); // Should be setInt64(5, ...) - } - // After the final chunk, we either encrypt a final, empty data - // chunk to get the final authentication tag or validate that final - // authentication tag. - cryptedPromises.push( - modeInstance[fn](finalChunk, mode.getNonce(this.iv, chunkIndexArray), adataTagArray) - ); - return util.concatUint8Array(await Promise.all(cryptedPromises)); + const reader = stream.getReader(data); + let chunkIndex = 0; + let latestPromise = Promise.resolve(); + let cryptedBytes = 0; + let queuedBytes = 0; + const iv = this.iv; + return new ReadableStream({ + async pull(controller) { + let chunk = await reader.readBytes(chunkSize + tagLengthIfDecrypting) || new Uint8Array(); + const finalChunk = chunk.subarray(chunk.length - tagLengthIfDecrypting); + chunk = chunk.subarray(0, chunk.length - tagLengthIfDecrypting); + let cryptedPromise; + let done; + if (!chunkIndex || chunk.length) { + reader.unshift(finalChunk); + cryptedPromise = modeInstance[fn](chunk, mode.getNonce(iv, chunkIndexArray), adataArray); + } else { + // After the last chunk, we either encrypt a final, empty + // data chunk to get the final authentication tag or + // validate that final authentication tag. + adataView.setInt32(13 + 4, cryptedBytes); // Should be setInt64(13, ...) + cryptedPromise = modeInstance[fn](finalChunk, mode.getNonce(iv, chunkIndexArray), adataTagArray); + done = true; + } + cryptedBytes += chunk.length - tagLengthIfDecrypting; + queuedBytes += chunk.length - tagLengthIfDecrypting; + latestPromise = latestPromise.then(() => cryptedPromise).then(crypted => { + if (crypted.length) controller.enqueue(crypted); + queuedBytes -= chunk.length; + }).catch(err => controller.error(err)); + // console.log(fn, done, queuedBytes, controller.desiredSize); + if (done || queuedBytes > controller.desiredSize) { + await latestPromise; // Respect backpressure + } + if (!done) { + adataView.setInt32(5 + 4, ++chunkIndex); // Should be setInt64(5, ...) + await this.pull(controller); + } else { + controller.close(); + } + } + }); } else { return modeInstance[fn](data, this.iv); } diff --git a/src/stream.js b/src/stream.js index 8eba2921..99aafebe 100644 --- a/src/stream.js +++ b/src/stream.js @@ -29,7 +29,7 @@ function transform(input, process = () => undefined, finish = () => undefined) { try { const { done, value } = await reader.read(); const result = await (!done ? process : finish)(value); - if (result) controller.enqueue(result); + if (result !== undefined) controller.enqueue(result); else if (!done) await this.pull(controller); // ??? Chrome bug? if (done) controller.close(); } catch(e) { @@ -40,8 +40,8 @@ function transform(input, process = () => undefined, finish = () => undefined) { } const result1 = process(input); const result2 = finish(undefined); - if (result1 && result2) return util.concatUint8Array([result1, result2]); - return result1 || result2; + if (result1 !== undefined && result2 !== undefined) return util.concat([result1, result2]); + return result1 !== undefined ? result1 : result2; } function tee(input) { @@ -136,16 +136,16 @@ Reader.prototype.readLine = async function() { while (!returnVal) { const { done, value } = await this.read(); if (done) { - if (buffer.length) return util.concatUint8Array(buffer); + if (buffer.length) return util.concat(buffer); return; } - const lineEndIndex = value.indexOf(10) + 1; // Position after the first "\n" + const lineEndIndex = value.indexOf('\n') + 1; if (lineEndIndex) { - returnVal = util.concatUint8Array(buffer.concat(value.subarray(0, lineEndIndex))); + returnVal = util.concat(buffer.concat(value.substr(0, lineEndIndex))); buffer = []; } if (lineEndIndex !== value.length) { - buffer.push(value.subarray(lineEndIndex)); + buffer.push(value.substr(lineEndIndex)); } } this.unshift(...buffer); @@ -166,13 +166,13 @@ Reader.prototype.readBytes = async function(length) { while (true) { const { done, value } = await this.read(); if (done) { - if (buffer.length) return util.concatUint8Array(buffer); + if (buffer.length) return util.concat(buffer); return; } buffer.push(value); bufferLength += value.length; if (bufferLength >= length) { - const bufferConcat = util.concatUint8Array(buffer); + const bufferConcat = util.concat(buffer); this.unshift(bufferConcat.subarray(length)); return bufferConcat.subarray(0, length); } @@ -207,7 +207,7 @@ function pullFrom(reader) { }; } -Reader.prototype.readToEnd = async function(join=util.concatUint8Array) { +Reader.prototype.readToEnd = async function(join=util.concat) { const result = []; while (true) { const { done, value } = await this.read(); diff --git a/src/util.js b/src/util.js index fe60b339..38ed6656 100644 --- a/src/util.js +++ b/src/util.js @@ -174,7 +174,7 @@ export default { * @returns {Uint8Array} An array of 8-bit integers */ b64_to_Uint8Array: function (base64) { - return b64.decode(util.str_to_Uint8Array(base64.replace(/-/g, '+').replace(/_/g, '/'))); + return b64.decode(base64.replace(/-/g, '+').replace(/_/g, '/')); }, /** @@ -281,6 +281,18 @@ export default { } }, + /** + * Concat a list of Uint8arrays or a list of Strings + * @param {Array} Array of Uint8Arrays/Strings to concatenate + * @returns {Uint8array|String} Concatenated array + */ + concat: function (arrays) { + if (util.isUint8Array(arrays[0])) { + return util.concatUint8Array(arrays); + } + return arrays.join(''); + }, + /** * Concat Uint8arrays * @param {Array} Array of Uint8Arrays to concatenate diff --git a/test/general/packet.js b/test/general/packet.js index a0ea4257..b77b62e2 100644 --- a/test/general/packet.js +++ b/test/general/packet.js @@ -145,8 +145,8 @@ describe("Packet", function() { return enc.encrypt(algo, key).then(async function() { await msg2.read(msg.write()); return msg2[0].decrypt(algo, key); - }).then(function() { - expect(msg2[0].packets[0].data).to.deep.equal(literal.data); + }).then(async function() { + expect(await openpgp.stream.readToEnd(msg2[0].packets[0].data)).to.deep.equal(literal.data); }); }); @@ -173,8 +173,8 @@ describe("Packet", function() { return enc.encrypt(algo, key).then(async function() { await msg2.read(msg.write()); return msg2[0].decrypt(algo, key); - }).then(function() { - expect(msg2[0].packets[0].data).to.deep.equal(literal.data); + }).then(async function() { + expect(await openpgp.stream.readToEnd(msg2[0].packets[0].data)).to.deep.equal(literal.data); }).finally(function() { openpgp.config.aead_protect = aead_protectVal; openpgp.config.aead_protect_version = aead_protect_versionVal; @@ -218,12 +218,12 @@ describe("Packet", function() { randomBytesStub.returns(resolves(iv)); return enc.encrypt(algo, key).then(async function() { - const data = msg.write(); - expect(data).to.deep.equal(packetBytes); + const [data, dataClone] = openpgp.stream.tee(msg.write()); + expect(await openpgp.stream.readToEnd(dataClone)).to.deep.equal(packetBytes); await msg2.read(data); return msg2[0].decrypt(algo, key); - }).then(function() { - expect(msg2[0].packets[0].data).to.deep.equal(literal.data); + }).then(async function() { + expect(await openpgp.stream.readToEnd(msg2[0].packets[0].data)).to.deep.equal(literal.data); }).finally(function() { openpgp.config.aead_protect = aead_protectVal; openpgp.config.aead_protect_version = aead_protect_versionVal; @@ -531,8 +531,8 @@ describe("Packet", function() { enc.packets.push(literal); await enc.encrypt(algo, key); - const data = msg.write(); - expect(data).to.deep.equal(packetBytes); + const [data, dataClone] = openpgp.stream.tee(msg.write()); + expect(await openpgp.stream.readToEnd(dataClone)).to.deep.equal(packetBytes); const msg2 = new openpgp.packet.List(); await msg2.read(data); @@ -610,8 +610,8 @@ describe("Packet", function() { enc.packets.push(literal); await enc.encrypt(algo, key); - const data = msg.write(); - expect(data).to.deep.equal(packetBytes); + const [data, dataClone] = openpgp.stream.tee(msg.write()); + expect(await openpgp.stream.readToEnd(dataClone)).to.deep.equal(packetBytes); const msg2 = new openpgp.packet.List(); await msg2.read(data); diff --git a/test/general/streaming.js b/test/general/streaming.js index 071892ac..70478c90 100644 --- a/test/general/streaming.js +++ b/test/general/streaming.js @@ -21,7 +21,7 @@ describe('Streaming', function() { data, passwords: ['test'], }); - const msgAsciiArmored = util.Uint8Array_to_str(await openpgp.stream.readToEnd(encrypted.data)); + const msgAsciiArmored = await openpgp.stream.readToEnd(encrypted.data); const message = await openpgp.message.readArmored(msgAsciiArmored); const decrypted = await openpgp.decrypt({ passwords: ['test'], @@ -48,7 +48,7 @@ describe('Streaming', function() { data, passwords: ['test'], }); - const msgAsciiArmored = util.Uint8Array_to_str(await openpgp.stream.readToEnd(encrypted.data)); + const msgAsciiArmored = await openpgp.stream.readToEnd(encrypted.data); const message = await openpgp.message.readArmored(msgAsciiArmored); const decrypted = await openpgp.decrypt({ passwords: ['test'], @@ -87,4 +87,40 @@ describe('Streaming', function() { expect(util.isStream(decrypted.data)).to.be.true; expect(await openpgp.stream.readToEnd(decrypted.data)).to.deep.equal(util.concatUint8Array(plaintext)); }); + + it('Encrypt and decrypt larger message roundtrip (draft04)', async function() { + let aead_protectValue = openpgp.config.aead_protect; + openpgp.config.aead_protect = true; + try { + let plaintext = []; + let i = 0; + const data = new ReadableStream({ + async pull(controller) { + if (i++ < 10) { + let randomBytes = await openpgp.crypto.random.getRandomBytes(1024); + controller.enqueue(randomBytes); + plaintext.push(randomBytes); + } else { + controller.close(); + } + } + }); + const encrypted = await openpgp.encrypt({ + data, + passwords: ['test'], + }); + + const msgAsciiArmored = encrypted.data; + const message = await openpgp.message.readArmored(msgAsciiArmored); + const decrypted = await openpgp.decrypt({ + passwords: ['test'], + message, + format: 'binary' + }); + expect(util.isStream(decrypted.data)).to.be.true; + expect(await openpgp.stream.readToEnd(decrypted.data)).to.deep.equal(util.concatUint8Array(plaintext)); + } finally { + openpgp.config.aead_protect = aead_protectValue; + } + }); });