Streaming AEAD

src/encoding/armor.js

@@ -119,7 +119,7 @@ function addheader(customComment) {
 /**
  * Calculates a checksum over the given data and returns it base64 encoded
  * @param {String} data Data to create a CRC-24 checksum for
- * @returns {Uint8Array} Base64 encoded checksum
+ * @returns {String} Base64 encoded checksum
  */
 function getCheckSum(data) {
   const crc = createcrc24(data);
@@ -201,9 +201,6 @@ function verifyHeaders(headers) {
  * @static
  */
 function dearmor(input) {
-  if (util.isString(input)) {
-    input = util.str_to_Uint8Array(util.encode_utf8(input));
-  }
   return new Promise(async (resolve, reject) => {
     try {
       const reSplit = /^-----[^-]+-----$/;
@@ -227,7 +224,7 @@ function dearmor(input) {
       const checksumVerified = getCheckSum(dataClone);
       data = stream.getReader(data).substream(); // Convert to Stream
       data = stream.transform(data, value => value, async () => {
-        const checksumVerifiedString = util.Uint8Array_to_str(await stream.readToEnd(checksumVerified));
+        const checksumVerifiedString = await stream.readToEnd(checksumVerified);
         if (checksum !== checksumVerifiedString && (checksum || config.checksum_required)) {
           throw new Error("Ascii armor integrity check on message failed: '" + checksum + "' should be '" +
                   checksumVerifiedString + "'");
@@ -236,7 +233,6 @@ function dearmor(input) {
       while (true) {
         let line = await reader.readLine();
         if (!line) break;
-        line = util.decode_utf8(util.Uint8Array_to_str(line));
         if (lineIndex++ === 0) {
           // trim string
           line = line.trim();
@@ -272,7 +268,7 @@ function dearmor(input) {
         } else {
           if (!reSplit.test(line)) {
             if (line[0] !== '=') {
-              controller.enqueue(util.str_to_Uint8Array(line));
+              controller.enqueue(line);
             } else {
               checksum = line.substr(1);
             }
@@ -365,7 +361,7 @@ function armor(messagetype, body, partindex, parttotal, customComment) {
       break;
   }
 
-  return util.concatUint8Array(result.map(part => (util.isString(part) ? util.str_to_Uint8Array(part) : part)));
+  return stream.concat(result);
 }
 
 export default {

src/encoding/base64.js

@@ -13,15 +13,13 @@
 
 /**
  * @requires stream
- * @requires util
  * @module encoding/base64
  */
 
 import stream from '../stream';
-import util from '../util';
 
-const b64s = util.str_to_Uint8Array('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'); // Standard radix-64
+const b64s = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; // Standard radix-64
-const b64u = util.str_to_Uint8Array('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'); // URL-safe radix-64
+const b64u = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'; // URL-safe radix-64
 
 /**
  * Convert binary array to radix-64
@@ -45,22 +43,22 @@ function s2r(t, u = false) {
     for (let n = 0; n < tl; n++) {
       c = value[n];
       if (s === 0) {
-        r.push(b64[(c >> 2) & 63]);
+        r.push(b64.charAt((c >> 2) & 63));
         a = (c & 3) << 4;
       } else if (s === 1) {
-        r.push(b64[a | ((c >> 4) & 15)]);
+        r.push(b64.charAt(a | ((c >> 4) & 15)));
         a = (c & 15) << 2;
       } else if (s === 2) {
-        r.push(b64[a | ((c >> 6) & 3)]);
+        r.push(b64.charAt(a | ((c >> 6) & 3)));
         l += 1;
         if ((l % 60) === 0 && !u) {
-          r.push(10); // "\n"
+          r.push("\n");
         }
-        r.push(b64[c & 63]);
+        r.push(b64.charAt(c & 63));
       }
       l += 1;
       if ((l % 60) === 0 && !u) {
-        r.push(10); // "\n"
+        r.push("\n");
       }
 
       s += 1;
@@ -68,27 +66,27 @@ function s2r(t, u = false) {
         s = 0;
       }
     }
-    return new Uint8Array(r);
+    return r.join('');
   }, () => {
     const r = [];
     if (s > 0) {
-      r.push(b64[a]);
+      r.push(b64.charAt(a));
       l += 1;
       if ((l % 60) === 0 && !u) {
-        r.push(10); // "\n"
+        r.push("\n");
       }
       if (!u) {
-        r.push(61); // "="
+        r.push('=');
         l += 1;
       }
     }
     if (s === 1 && !u) {
       if ((l % 60) === 0 && !u) {
-        r.push(10); // "\n"
+        r.push("\n");
       }
-      r.push(61); // "="
+      r.push('=');
     }
-    return new Uint8Array(r);
+    return r.join('');
   });
 }
 
@@ -111,7 +109,7 @@ function r2s(t, u) {
     const r = [];
     const tl = value.length;
     for (let n = 0; n < tl; n++) {
-      c = b64.indexOf(value[n]);
+      c = b64.indexOf(value.charAt(n));
       if (c >= 0) {
         if (s) {
           r.push(a | ((c >> (6 - s)) & 255));

src/openpgp.js

@@ -323,12 +323,15 @@ export function encrypt({ data, dataType, publicKeys, privateKeys, passwords, se
     message = message.compress(compression);
     return message.encrypt(publicKeys, passwords, sessionKey, wildcard, date, toUserId);
 
-  }).then(encrypted => {
+  }).then(async encrypted => {
+    let message = encrypted.message;
     if (armor) {
-      result.data = encrypted.message.armor();
+      message = message.armor();
-    } else {
-      result.message = encrypted.message;
     }
+    if (util.isStream(message) && !util.isStream(data)) {
+      message = await stream.readToEnd(message);
+    }
+    result[armor ? 'data' : 'message'] = message;
     if (returnSessionKey) {
       result.sessionKey = encrypted.sessionKey;
     }
@@ -597,17 +600,14 @@ async function parseMessage(message, format, asStream) {
   let data;
   if (format === 'binary') {
     data = message.getLiteralData();
-    if (!asStream && util.isStream(data)) {
-      data = await stream.readToEnd(data);
-    }
   } else if (format === 'utf8') {
     data = message.getText();
-    if (!asStream && util.isStream(data)) {
-      data = await stream.readToEnd(data, chunks => chunks.join(''));
-    }
   } else {
     throw new Error('Invalid format');
   }
+  if (!asStream && util.isStream(data)) {
+    data = await stream.readToEnd(data);
+  }
   const filename = message.getFilename();
   return { data, filename };
 }

src/packet/literal.js

@@ -75,7 +75,7 @@ Literal.prototype.getText = function() {
       // decode UTF8 and normalize EOL to \n
       const normalized = normalize(text);
       // if last two bytes are \r\n or an UTF8 sequence, return them immediately
-      if (text.length >= 2 && normalize(text.slice(-2)).length === 1) {
+      if (text.length >= 2 && text.slice(-2) !== normalized.slice(-2)) {
         lastChar = '';
         return normalized;
       }

src/packet/sym_encrypted_aead_protected.js

@@ -19,12 +19,14 @@
  * @requires config
  * @requires crypto
  * @requires enums
+ * @requires stream
  * @requires util
  */
 
 import config from '../config';
 import crypto from '../crypto';
 import enums from '../enums';
+import stream from '../stream';
 import util from '../util';
 
 const VERSION = 1; // A one-octet version number of the data packet.
@@ -55,23 +57,21 @@ export default SymEncryptedAEADProtected;
 /**
  * Parse an encrypted payload of bytes in the order: version, IV, ciphertext (see specification)
  */
-SymEncryptedAEADProtected.prototype.read = function (bytes) {
+SymEncryptedAEADProtected.prototype.read = async function (bytes) {
-  let offset = 0;
+  const reader = stream.getReader(bytes);
-  if (bytes[offset] !== VERSION) { // The only currently defined value is 1.
+  if (await reader.readByte() !== VERSION) { // The only currently defined value is 1.
     throw new Error('Invalid packet version.');
   }
-  offset++;
   if (config.aead_protect_version === 4) {
-    this.cipherAlgo = bytes[offset++];
+    this.cipherAlgo = await reader.readByte();
-    this.aeadAlgo = bytes[offset++];
+    this.aeadAlgo = await reader.readByte();
-    this.chunkSizeByte = bytes[offset++];
+    this.chunkSizeByte = await reader.readByte();
   } else {
     this.aeadAlgo = enums.aead.experimental_gcm;
   }
   const mode = crypto[enums.read(enums.aead, this.aeadAlgo)];
-  this.iv = bytes.subarray(offset, mode.ivLength + offset);
+  this.iv = await reader.readBytes(mode.ivLength);
-  offset += mode.ivLength;
+  this.encrypted = reader.substream();
-  this.encrypted = bytes.subarray(offset, bytes.length);
 };
 
 /**
@@ -93,15 +93,10 @@ SymEncryptedAEADProtected.prototype.write = function () {
  * @async
  */
 SymEncryptedAEADProtected.prototype.decrypt = async function (sessionKeyAlgorithm, key) {
-  const mode = crypto[enums.read(enums.aead, this.aeadAlgo)];
+  if (config.aead_protect_version !== 4) {
-  if (config.aead_protect_version === 4) {
-    const data = this.encrypted.subarray(0, -mode.tagLength);
-    const authTag = this.encrypted.subarray(-mode.tagLength);
-    await this.packets.read(await this.crypt('decrypt', key, data, authTag));
-  } else {
     this.cipherAlgo = enums.write(enums.symmetric, sessionKeyAlgorithm);
-    await this.packets.read(await this.crypt('decrypt', key, this.encrypted));
   }
+  await this.packets.read(await this.crypt('decrypt', key, this.encrypted));
   return true;
 };
 
@@ -119,7 +114,7 @@ SymEncryptedAEADProtected.prototype.encrypt = async function (sessionKeyAlgorith
   this.iv = await crypto.random.getRandomBytes(mode.ivLength); // generate new random IV
   this.chunkSizeByte = config.aead_chunk_size_byte;
   const data = this.packets.write();
-  this.encrypted = await this.crypt('encrypt', key, data, data.subarray(0, 0));
+  this.encrypted = await this.crypt('encrypt', key, data);
 };
 
 /**
@@ -127,11 +122,10 @@ SymEncryptedAEADProtected.prototype.encrypt = async function (sessionKeyAlgorith
  * @param  {encrypt|decrypt} fn      Whether to encrypt or decrypt
  * @param  {Uint8Array} key          The session key used to en/decrypt the payload
  * @param  {Uint8Array} data         The data to en/decrypt
- * @param  {Uint8Array} finalChunk   For encryption: empty final chunk; for decryption: final authentication tag
  * @returns {Promise<Uint8Array>}
  * @async
  */
-SymEncryptedAEADProtected.prototype.crypt = async function (fn, key, data, finalChunk) {
+SymEncryptedAEADProtected.prototype.crypt = async function (fn, key, data) {
   const cipher = enums.read(enums.symmetric, this.cipherAlgo);
   const mode = crypto[enums.read(enums.aead, this.aeadAlgo)];
   const modeInstance = await mode(cipher, key);
@@ -144,24 +138,48 @@ SymEncryptedAEADProtected.prototype.crypt = async function (fn, key, data, final
     const adataView = new DataView(adataBuffer);
     const chunkIndexArray = new Uint8Array(adataBuffer, 5, 8);
     adataArray.set([0xC0 | this.tag, this.version, this.cipherAlgo, this.aeadAlgo, this.chunkSizeByte], 0);
-    adataView.setInt32(13 + 4, data.length - tagLengthIfDecrypting * Math.ceil(data.length / chunkSize)); // Should be setInt64(13, ...)
+    const reader = stream.getReader(data);
-    const cryptedPromises = [];
+    let chunkIndex = 0;
-    for (let chunkIndex = 0; chunkIndex === 0 || data.length;) {
+    let latestPromise = Promise.resolve();
-      cryptedPromises.push(
+    let cryptedBytes = 0;
-        modeInstance[fn](data.subarray(0, chunkSize), mode.getNonce(this.iv, chunkIndexArray), adataArray)
+    let queuedBytes = 0;
-      );
+    const iv = this.iv;
-      // We take a chunk of data, en/decrypt it, and shift `data` to the
+    return new ReadableStream({
-      // next chunk.
+      async pull(controller) {
-      data = data.subarray(chunkSize);
+        let chunk = await reader.readBytes(chunkSize + tagLengthIfDecrypting) || new Uint8Array();
-      adataView.setInt32(5 + 4, ++chunkIndex); // Should be setInt64(5, ...)
+        const finalChunk = chunk.subarray(chunk.length - tagLengthIfDecrypting);
-    }
+        chunk = chunk.subarray(0, chunk.length - tagLengthIfDecrypting);
-    // After the final chunk, we either encrypt a final, empty data
+        let cryptedPromise;
-    // chunk to get the final authentication tag or validate that final
+        let done;
-    // authentication tag.
+        if (!chunkIndex || chunk.length) {
-    cryptedPromises.push(
+          reader.unshift(finalChunk);
-      modeInstance[fn](finalChunk, mode.getNonce(this.iv, chunkIndexArray), adataTagArray)
+          cryptedPromise = modeInstance[fn](chunk, mode.getNonce(iv, chunkIndexArray), adataArray);
-    );
+        } else {
-    return util.concatUint8Array(await Promise.all(cryptedPromises));
+          // After the last chunk, we either encrypt a final, empty
+          // data chunk to get the final authentication tag or
+          // validate that final authentication tag.
+          adataView.setInt32(13 + 4, cryptedBytes); // Should be setInt64(13, ...)
+          cryptedPromise = modeInstance[fn](finalChunk, mode.getNonce(iv, chunkIndexArray), adataTagArray);
+          done = true;
+        }
+        cryptedBytes += chunk.length - tagLengthIfDecrypting;
+        queuedBytes += chunk.length - tagLengthIfDecrypting;
+        latestPromise = latestPromise.then(() => cryptedPromise).then(crypted => {
+          if (crypted.length) controller.enqueue(crypted);
+          queuedBytes -= chunk.length;
+        }).catch(err => controller.error(err));
+        // console.log(fn, done, queuedBytes, controller.desiredSize);
+        if (done || queuedBytes > controller.desiredSize) {
+          await latestPromise; // Respect backpressure
+        }
+        if (!done) {
+          adataView.setInt32(5 + 4, ++chunkIndex); // Should be setInt64(5, ...)
+          await this.pull(controller);
+        } else {
+          controller.close();
+        }
+      }
+    });
   } else {
     return modeInstance[fn](data, this.iv);
   }

src/stream.js

@@ -29,7 +29,7 @@ function transform(input, process = () => undefined, finish = () => undefined) {
         try {
           const { done, value } = await reader.read();
           const result = await (!done ? process : finish)(value);
-          if (result) controller.enqueue(result);
+          if (result !== undefined) controller.enqueue(result);
           else if (!done) await this.pull(controller); // ??? Chrome bug?
           if (done) controller.close();
         } catch(e) {
@@ -40,8 +40,8 @@ function transform(input, process = () => undefined, finish = () => undefined) {
   }
   const result1 = process(input);
   const result2 = finish(undefined);
-  if (result1 && result2) return util.concatUint8Array([result1, result2]);
+  if (result1 !== undefined && result2 !== undefined) return util.concat([result1, result2]);
-  return result1 || result2;
+  return result1 !== undefined ? result1 : result2;
 }
 
 function tee(input) {
@@ -136,16 +136,16 @@ Reader.prototype.readLine = async function() {
   while (!returnVal) {
     const { done, value } = await this.read();
     if (done) {
-      if (buffer.length) return util.concatUint8Array(buffer);
+      if (buffer.length) return util.concat(buffer);
       return;
     }
-    const lineEndIndex = value.indexOf(10) + 1; // Position after the first "\n"
+    const lineEndIndex = value.indexOf('\n') + 1;
     if (lineEndIndex) {
-      returnVal = util.concatUint8Array(buffer.concat(value.subarray(0, lineEndIndex)));
+      returnVal = util.concat(buffer.concat(value.substr(0, lineEndIndex)));
       buffer = [];
     }
     if (lineEndIndex !== value.length) {
-      buffer.push(value.subarray(lineEndIndex));
+      buffer.push(value.substr(lineEndIndex));
     }
   }
   this.unshift(...buffer);
@@ -166,13 +166,13 @@ Reader.prototype.readBytes = async function(length) {
   while (true) {
     const { done, value } = await this.read();
     if (done) {
-      if (buffer.length) return util.concatUint8Array(buffer);
+      if (buffer.length) return util.concat(buffer);
       return;
     }
     buffer.push(value);
     bufferLength += value.length;
     if (bufferLength >= length) {
-      const bufferConcat = util.concatUint8Array(buffer);
+      const bufferConcat = util.concat(buffer);
       this.unshift(bufferConcat.subarray(length));
       return bufferConcat.subarray(0, length);
     }
@@ -207,7 +207,7 @@ function pullFrom(reader) {
   };
 }
 
-Reader.prototype.readToEnd = async function(join=util.concatUint8Array) {
+Reader.prototype.readToEnd = async function(join=util.concat) {
   const result = [];
   while (true) {
     const { done, value } = await this.read();

src/util.js

@@ -174,7 +174,7 @@ export default {
    * @returns {Uint8Array} An array of 8-bit integers
    */
   b64_to_Uint8Array: function (base64) {
-    return b64.decode(util.str_to_Uint8Array(base64.replace(/-/g, '+').replace(/_/g, '/')));
+    return b64.decode(base64.replace(/-/g, '+').replace(/_/g, '/'));
   },
 
   /**
@@ -281,6 +281,18 @@ export default {
     }
   },
 
+  /**
+   * Concat a list of Uint8arrays or a list of Strings
+   * @param {Array<Uint8array|String>} Array of Uint8Arrays/Strings to concatenate
+   * @returns {Uint8array|String} Concatenated array
+   */
+  concat: function (arrays) {
+    if (util.isUint8Array(arrays[0])) {
+      return util.concatUint8Array(arrays);
+    }
+    return arrays.join('');
+  },
+
   /**
    * Concat Uint8arrays
    * @param {Array<Uint8array>} Array of Uint8Arrays to concatenate

test/general/packet.js

@@ -145,8 +145,8 @@ describe("Packet", function() {
     return enc.encrypt(algo, key).then(async function() {
       await msg2.read(msg.write());
       return msg2[0].decrypt(algo, key);
-    }).then(function() {
+    }).then(async function() {
-      expect(msg2[0].packets[0].data).to.deep.equal(literal.data);
+      expect(await openpgp.stream.readToEnd(msg2[0].packets[0].data)).to.deep.equal(literal.data);
     });
   });
 
@@ -173,8 +173,8 @@ describe("Packet", function() {
     return enc.encrypt(algo, key).then(async function() {
       await msg2.read(msg.write());
       return msg2[0].decrypt(algo, key);
-    }).then(function() {
+    }).then(async function() {
-      expect(msg2[0].packets[0].data).to.deep.equal(literal.data);
+      expect(await openpgp.stream.readToEnd(msg2[0].packets[0].data)).to.deep.equal(literal.data);
     }).finally(function() {
       openpgp.config.aead_protect = aead_protectVal;
       openpgp.config.aead_protect_version = aead_protect_versionVal;
@@ -218,12 +218,12 @@ describe("Packet", function() {
     randomBytesStub.returns(resolves(iv));
 
     return enc.encrypt(algo, key).then(async function() {
-      const data = msg.write();
+      const [data, dataClone] = openpgp.stream.tee(msg.write());
-      expect(data).to.deep.equal(packetBytes);
+      expect(await openpgp.stream.readToEnd(dataClone)).to.deep.equal(packetBytes);
       await msg2.read(data);
       return msg2[0].decrypt(algo, key);
-    }).then(function() {
+    }).then(async function() {
-      expect(msg2[0].packets[0].data).to.deep.equal(literal.data);
+      expect(await openpgp.stream.readToEnd(msg2[0].packets[0].data)).to.deep.equal(literal.data);
     }).finally(function() {
       openpgp.config.aead_protect = aead_protectVal;
       openpgp.config.aead_protect_version = aead_protect_versionVal;
@@ -531,8 +531,8 @@ describe("Packet", function() {
       enc.packets.push(literal);
       await enc.encrypt(algo, key);
 
-      const data = msg.write();
+      const [data, dataClone] = openpgp.stream.tee(msg.write());
-      expect(data).to.deep.equal(packetBytes);
+      expect(await openpgp.stream.readToEnd(dataClone)).to.deep.equal(packetBytes);
 
       const msg2 = new openpgp.packet.List();
       await msg2.read(data);
@@ -610,8 +610,8 @@ describe("Packet", function() {
       enc.packets.push(literal);
       await enc.encrypt(algo, key);
 
-      const data = msg.write();
+      const [data, dataClone] = openpgp.stream.tee(msg.write());
-      expect(data).to.deep.equal(packetBytes);
+      expect(await openpgp.stream.readToEnd(dataClone)).to.deep.equal(packetBytes);
 
       const msg2 = new openpgp.packet.List();
       await msg2.read(data);

test/general/streaming.js

@@ -21,7 +21,7 @@ describe('Streaming', function() {
       data,
       passwords: ['test'],
     });
-    const msgAsciiArmored = util.Uint8Array_to_str(await openpgp.stream.readToEnd(encrypted.data));
+    const msgAsciiArmored = await openpgp.stream.readToEnd(encrypted.data);
     const message = await openpgp.message.readArmored(msgAsciiArmored);
     const decrypted = await openpgp.decrypt({
       passwords: ['test'],
@@ -48,7 +48,7 @@ describe('Streaming', function() {
       data,
       passwords: ['test'],
     });
-    const msgAsciiArmored = util.Uint8Array_to_str(await openpgp.stream.readToEnd(encrypted.data));
+    const msgAsciiArmored = await openpgp.stream.readToEnd(encrypted.data);
     const message = await openpgp.message.readArmored(msgAsciiArmored);
     const decrypted = await openpgp.decrypt({
       passwords: ['test'],
@@ -87,4 +87,40 @@ describe('Streaming', function() {
     expect(util.isStream(decrypted.data)).to.be.true;
     expect(await openpgp.stream.readToEnd(decrypted.data)).to.deep.equal(util.concatUint8Array(plaintext));
   });
+
+  it('Encrypt and decrypt larger message roundtrip (draft04)', async function() {
+    let aead_protectValue = openpgp.config.aead_protect;
+    openpgp.config.aead_protect = true;
+    try {
+      let plaintext = [];
+      let i = 0;
+      const data = new ReadableStream({
+        async pull(controller) {
+          if (i++ < 10) {
+            let randomBytes = await openpgp.crypto.random.getRandomBytes(1024);
+            controller.enqueue(randomBytes);
+            plaintext.push(randomBytes);
+          } else {
+            controller.close();
+          }
+        }
+      });
+      const encrypted = await openpgp.encrypt({
+        data,
+        passwords: ['test'],
+      });
+
+      const msgAsciiArmored = encrypted.data;
+      const message = await openpgp.message.readArmored(msgAsciiArmored);
+      const decrypted = await openpgp.decrypt({
+        passwords: ['test'],
+        message,
+        format: 'binary'
+      });
+      expect(util.isStream(decrypted.data)).to.be.true;
+      expect(await openpgp.stream.readToEnd(decrypted.data)).to.deep.equal(util.concatUint8Array(plaintext));
+    } finally {
+      openpgp.config.aead_protect = aead_protectValue;
+    }
+  });
 });