Streaming encryption (Web)

src/crypto/hash/index.js

@@ -13,6 +13,7 @@
 
 import Rusha from 'rusha';
 import { SHA256 } from 'asmcrypto.js/src/hash/sha256/exports';
+import sha1 from 'hash.js/lib/hash/sha/1';
 import sha224 from 'hash.js/lib/hash/sha/224';
 import sha384 from 'hash.js/lib/hash/sha/384';
 import sha512 from 'hash.js/lib/hash/sha/512';
@@ -34,7 +35,14 @@ function node_hash(type) {
 
 function hashjs_hash(hash) {
   return function(data) {
-    return util.hex_to_Uint8Array(hash().update(data).digest('hex'));
+    const hashInstance = hash();
+    return data.transform((done, value) => {
+      if (!done) {
+        hashInstance.update(value);
+      } else {
+        return util.hex_to_Uint8Array(hashInstance.digest('hex'));
+      }
+    });
   };
 }
 
@@ -52,9 +60,10 @@ if (nodeCrypto) { // Use Node native crypto for all hash functions
 } else { // Use JS fallbacks
   hash_fns = {
     md5: md5,
-    sha1: function(data) {
+    sha1: hashjs_hash(sha1),
+    /*sha1: function(data) {
       return util.hex_to_Uint8Array(rusha.digest(data));
-    },
+    },*/
     sha224: hashjs_hash(sha224),
     sha256: SHA256.bytes,
     sha384: hashjs_hash(sha384),

src/encoding/armor.js

@@ -117,12 +117,15 @@ function addheader(customComment) {
 /**
  * Calculates a checksum over the given data and returns it base64 encoded
  * @param {String} data Data to create a CRC-24 checksum for
- * @returns {String} Base64 encoded checksum
+ * @returns {Uint8Array} Base64 encoded checksum
  */
 function getCheckSum(data) {
-  const c = createcrc24(data);
+  const crc = createcrc24(data);
-  const bytes = new Uint8Array([c >> 16, (c >> 8) & 0xFF, c & 0xFF]);
+  return base64.encode(crc);
-  return base64.encode(bytes);
+}
+
+function getCheckSumString(data) {
+  return util.Uint8Array_to_str(getCheckSum(data));
 }
 
 /**
@@ -133,10 +136,11 @@ function getCheckSum(data) {
  * @returns {Boolean} True if the given checksum is correct; otherwise false
  */
 function verifyCheckSum(data, checksum) {
-  const c = getCheckSum(data);
+  const c = getCheckSumString(data);
   const d = checksum;
   return c[0] === d[0] && c[1] === d[1] && c[2] === d[2] && c[3] === d[3];
 }
+
 /**
  * Internal function to calculate a CRC-24 checksum over a given string (data)
  * @param {String} data Data to create a CRC-24 checksum for
@@ -179,11 +183,16 @@ const crc_table = [
 
 function createcrc24(input) {
   let crc = 0xB704CE;
-
+  return input.transform((done, value) => {
-  for (let index = 0; index < input.length; index++) {
+    if (!done) {
-    crc = (crc << 8) ^ crc_table[((crc >> 16) ^ input[index]) & 0xff];
+      for (let index = 0; index < value.length; index++) {
+        crc = (crc << 8) ^ crc_table[((crc >> 16) ^ value[index]) & 0xff];
       }
-  return crc & 0xffffff;
+    } else {
+      crc &= 0xffffff;
+      return new Uint8Array([crc >> 16, (crc >> 8) & 0xFF, crc & 0xFF]);
+    }
+  });
 }
 
 /**
@@ -315,7 +324,7 @@ function dearmor(text) {
   if (!verifyCheckSum(result.data, checksum) && (checksum || config.checksum_required)) {
     // will NOT throw error if checksum is empty AND checksum is not required (GPG compatibility)
     throw new Error("Ascii armor integrity check on message failed: '" + checksum + "' should be '" +
-      getCheckSum(result.data) + "'");
+      getCheckSumString(result.data) + "'");
   }
 
   verifyHeaders(result.headers);
@@ -335,63 +344,70 @@ function dearmor(text) {
  * @static
  */
 function armor(messagetype, body, partindex, parttotal, customComment) {
+  let text;
+  if (messagetype === enums.armor.signed) {
+    text = body.text;
+    body = body.data;
+  }
+  let bodyClone;
+  [body, bodyClone] = body.tee();
   const result = [];
   switch (messagetype) {
     case enums.armor.multipart_section:
       result.push("-----BEGIN PGP MESSAGE, PART " + partindex + "/" + parttotal + "-----\r\n");
       result.push(addheader(customComment));
       result.push(base64.encode(body));
-      result.push("\r\n=" + getCheckSum(body) + "\r\n");
+      result.push("\r\n=", getCheckSum(bodyClone), "\r\n");
       result.push("-----END PGP MESSAGE, PART " + partindex + "/" + parttotal + "-----\r\n");
       break;
     case enums.armor.multipart_last:
       result.push("-----BEGIN PGP MESSAGE, PART " + partindex + "-----\r\n");
       result.push(addheader(customComment));
       result.push(base64.encode(body));
-      result.push("\r\n=" + getCheckSum(body) + "\r\n");
+      result.push("\r\n=", getCheckSum(bodyClone), "\r\n");
       result.push("-----END PGP MESSAGE, PART " + partindex + "-----\r\n");
       break;
     case enums.armor.signed:
       result.push("\r\n-----BEGIN PGP SIGNED MESSAGE-----\r\n");
       result.push("Hash: " + body.hash + "\r\n\r\n");
-      result.push(body.text.replace(/^-/mg, "- -"));
+      result.push(text.replace(/^-/mg, "- -"));
       result.push("\r\n-----BEGIN PGP SIGNATURE-----\r\n");
       result.push(addheader(customComment));
-      result.push(base64.encode(body.data));
+      result.push(base64.encode(body));
-      result.push("\r\n=" + getCheckSum(body.data) + "\r\n");
+      result.push("\r\n=", getCheckSum(bodyClone), "\r\n");
       result.push("-----END PGP SIGNATURE-----\r\n");
       break;
     case enums.armor.message:
       result.push("-----BEGIN PGP MESSAGE-----\r\n");
       result.push(addheader(customComment));
       result.push(base64.encode(body));
-      result.push("\r\n=" + getCheckSum(body) + "\r\n");
+      result.push("\r\n=", getCheckSum(bodyClone), "\r\n");
       result.push("-----END PGP MESSAGE-----\r\n");
       break;
     case enums.armor.public_key:
       result.push("-----BEGIN PGP PUBLIC KEY BLOCK-----\r\n");
       result.push(addheader(customComment));
       result.push(base64.encode(body));
-      result.push("\r\n=" + getCheckSum(body) + "\r\n");
+      result.push("\r\n=", getCheckSum(bodyClone), "\r\n");
       result.push("-----END PGP PUBLIC KEY BLOCK-----\r\n\r\n");
       break;
     case enums.armor.private_key:
       result.push("-----BEGIN PGP PRIVATE KEY BLOCK-----\r\n");
       result.push(addheader(customComment));
       result.push(base64.encode(body));
-      result.push("\r\n=" + getCheckSum(body) + "\r\n");
+      result.push("\r\n=", getCheckSum(bodyClone), "\r\n");
       result.push("-----END PGP PRIVATE KEY BLOCK-----\r\n");
       break;
     case enums.armor.signature:
       result.push("-----BEGIN PGP SIGNATURE-----\r\n");
       result.push(addheader(customComment));
       result.push(base64.encode(body));
-      result.push("\r\n=" + getCheckSum(body) + "\r\n");
+      result.push("\r\n=", getCheckSum(bodyClone), "\r\n");
       result.push("-----END PGP SIGNATURE-----\r\n");
       break;
   }
 
-  return result.join('');
+  return util.concatUint8Array(result.map(part => (util.isString(part) ? util.str_to_Uint8Array(part) : part)));
 }
 
 export default {

src/encoding/base64.js

@@ -12,9 +12,12 @@
  */
 
 /**
+ * @requires util
  * @module encoding/base64
  */
 
+import util from '../util';
+
 const b64s = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; // Standard radix-64
 const b64u = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'; // URL-safe radix-64
 
@@ -30,14 +33,17 @@ function s2r(t, u = false) {
   const b64 = u ? b64u : b64s;
   let a;
   let c;
-  let n;
+
-  const r = [];
   let l = 0;
   let s = 0;
-  const tl = t.length;
 
-  for (n = 0; n < tl; n++) {
+  return t.transform((done, value) => {
-    c = t[n];
+    const r = [];
+
+    if (!done) {
+      const tl = value.length;
+      for (let n = 0; n < tl; n++) {
+        c = value[n];
         if (s === 0) {
           r.push(b64.charAt((c >> 2) & 63));
           a = (c & 3) << 4;
@@ -62,6 +68,7 @@ function s2r(t, u = false) {
           s = 0;
         }
       }
+    } else {
       if (s > 0) {
         r.push(b64.charAt(a));
         l += 1;
@@ -79,7 +86,9 @@ function s2r(t, u = false) {
         }
         r.push('=');
       }
-  return r.join('');
+    }
+    return util.str_to_Uint8Array(r.join(''));
+  });
 }
 
 /**

src/index.js

@@ -100,6 +100,12 @@ export { default as KDFParams } from './type/kdf_params';
  */
 export { default as OID } from './type/oid';
 
+/**
+ * @see module:type/oid
+ * @name module:openpgp.OID
+ */
+export { default as Stream } from './type/stream';
+
 /**
  * @see module:encoding/armor
  * @name module:openpgp.armor

src/message.js

@@ -674,7 +674,7 @@ export function fromText(text, filename, date=new Date(), type='utf8') {
  * @static
  */
 export function fromBinary(bytes, filename, date=new Date(), type='binary') {
-  if (!util.isUint8Array(bytes)) {
+  if (!util.isUint8Array(bytes) && !util.isStream(bytes)) {
     throw new Error('Data must be in the form of a Uint8Array');
   }
 

src/openpgp.js

@@ -536,8 +536,8 @@ function checkBinary(data, name) {
   }
 }
 function checkData(data, name) {
-  if (!util.isUint8Array(data) && !util.isString(data)) {
+  if (!util.isUint8Array(data) && !util.isString(data) && !util.isStream(data)) {
-    throw new Error('Parameter [' + (name || 'data') + '] must be of type String or Uint8Array');
+    throw new Error('Parameter [' + (name || 'data') + '] must be of type String, Uint8Array or ReadableStream');
   }
 }
 function checkMessage(message) {
@@ -573,7 +573,7 @@ function toArray(param) {
  */
 function createMessage(data, filename, date=new Date(), type) {
   let msg;
-  if (util.isUint8Array(data)) {
+  if (util.isUint8Array(data) || util.isStream(data)) {
     msg = messageLib.fromBinary(data, filename, date, type);
   } else if (util.isString(data)) {
     msg = messageLib.fromText(data, filename, date, type);

src/packet/sym_encrypted_integrity_protected.js

@@ -22,7 +22,9 @@
  * @requires util
  */
 
-import { AES_CFB } from 'asmcrypto.js/src/aes/cfb/exports';
+import { _AES_asm_instance, _AES_heap_instance } from 'asmcrypto.js/src/aes/exports';
+import { AES_CFB, AES_CFB_Decrypt, AES_CFB_Encrypt } from 'asmcrypto.js/src/aes/cfb/exports';
+
 import crypto from '../crypto';
 import enums from '../enums';
 import util from '../util';
@@ -89,12 +91,12 @@ SymEncryptedIntegrityProtected.prototype.encrypt = async function (sessionKeyAlg
   const prefix = util.concatUint8Array([prefixrandom, repeat]);
   const mdc = new Uint8Array([0xD3, 0x14]); // modification detection code packet
 
-  let tohash = util.concatUint8Array([bytes, mdc]);
+  let [tohash, tohashClone] = util.concatUint8Array([bytes, mdc]).tee();
-  const hash = crypto.hash.sha1(util.concatUint8Array([prefix, tohash]));
+  const hash = crypto.hash.sha1(util.concatUint8Array([prefix, tohashClone]));
   tohash = util.concatUint8Array([tohash, hash]);
 
   if (sessionKeyAlgorithm.substr(0, 3) === 'aes') { // AES optimizations. Native code for node, asmCrypto for browser.
-    this.encrypted = aesEncrypt(sessionKeyAlgorithm, prefix, tohash, key);
+    this.encrypted = aesEncrypt(sessionKeyAlgorithm, util.concatUint8Array([prefix, tohash]), key);
   } else {
     this.encrypted = crypto.cfb.encrypt(prefixrandom, sessionKeyAlgorithm, tohash, key, false);
     this.encrypted = this.encrypted.subarray(0, prefix.length + tohash.length);
@@ -144,11 +146,17 @@ export default SymEncryptedIntegrityProtected;
 //////////////////////////
 
 
-function aesEncrypt(algo, prefix, pt, key) {
+function aesEncrypt(algo, pt, key) {
   if (nodeCrypto) { // Node crypto library.
-    return nodeEncrypt(algo, prefix, pt, key);
+    return nodeEncrypt(algo, pt, key);
   } // asm.js fallback
-  return AES_CFB.encrypt(util.concatUint8Array([prefix, pt]), key);
+  const cfb = new AES_CFB_Encrypt(key, undefined, _AES_heap_instance, _AES_asm_instance);
+  return pt.transform((done, value) => {
+    if (!done) {
+      return cfb.process(value).result;
+    }
+    return cfb.finish().result;
+  });
 }
 
 function aesDecrypt(algo, ct, key) {

src/type/stream.js

@@ -0,0 +1,87 @@
+import util from '../util';
+
+function concat(arrays) {
+  const readers = arrays.map(entry => entry.getReader());
+  let current = 0;
+  return new ReadableStream({
+    async pull(controller) {
+      const { done, value } = await readers[current].read();
+      if (!done) {
+        controller.enqueue(value);
+      } else if (++current === arrays.length) {
+        controller.close();
+      } else {
+        await this.pull(controller); // ??? Chrome bug?
+      }
+    }
+  });
+}
+
+export default { concat };
+
+
+/*const readerAcquiredMap = new Map();
+
+const _getReader = ReadableStream.prototype.getReader;
+ReadableStream.prototype.getReader = function() {
+  if (readerAcquiredMap.has(this)) {
+    console.error(readerAcquiredMap.get(this));
+  } else {
+    readerAcquiredMap.set(this, new Error('Reader for this ReadableStream already acquired here.'));
+  }
+  return _getReader.apply(this, arguments);
+};*/
+
+
+ReadableStream.prototype.transform = function(fn) {
+  const reader = this.getReader();
+  return new ReadableStream({
+    async pull(controller) {
+      const { done, value } = await reader.read();
+      const result = fn(done, value);
+      if (result) controller.enqueue(result);
+      if (done) controller.close();
+      if (!done && !result) await this.pull(controller); // ??? Chrome bug?
+    }
+  });
+};
+
+ReadableStream.prototype.readToEnd = async function() {
+  const reader = this.getReader();
+  const result = [];
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    result.push(value);
+  }
+  return util.concatUint8Array(result);
+};
+
+
+Uint8Array.prototype.getReader = function() {
+  let doneReading = false;
+  return {
+    read: async () => {
+      if (doneReading) {
+        return { value: undefined, done: true };
+      }
+      doneReading = true;
+      return { value: this, done: false };
+    }
+  };
+};
+
+Uint8Array.prototype.transform = function(fn) {
+  const result1 = fn(false, this);
+  const result2 = fn(true, undefined);
+  if (result1 && result2) return util.concatUint8Array([result1, result2]);
+  return result1 || result2;
+};
+
+Uint8Array.prototype.tee = function() {
+  return [this, this];
+};
+
+Uint8Array.prototype.readToEnd = async function() {
+  return this;
+};

src/util.js

@@ -29,6 +29,7 @@ import rfc2822 from 'address-rfc2822';
 import config from './config';
 import util from './util'; // re-import module to access util functions
 import b64 from './encoding/base64';
+import Stream from './type/stream';
 
 const isIE11 = typeof navigator !== 'undefined' && !!navigator.userAgent.match(/Trident\/7\.0.*rv:([0-9.]+).*\).*Gecko$/);
 
@@ -45,6 +46,10 @@ export default {
     return Uint8Array.prototype.isPrototypeOf(data);
   },
 
+  isStream: function(data) {
+    return ReadableStream.prototype.isPrototypeOf(data);
+  },
+
   /**
    * Get transferable objects to pass buffers with zero copy (similar to "pass by reference" in C++)
    *   See: https://developer.mozilla.org/en-US/docs/Web/API/Worker/postMessage
@@ -282,6 +287,9 @@ export default {
   concatUint8Array: function (arrays) {
     let totalLength = 0;
     for (let i = 0; i < arrays.length; i++) {
+      if (util.isStream(arrays[i])) {
+        return Stream.concat(arrays);
+      }
       if (!util.isUint8Array(arrays[i])) {
         throw new Error('concatUint8Array: Data must be in the form of a Uint8Array');
       }
@@ -409,6 +417,14 @@ export default {
     }
   },
 
+  print_entire_stream: function (str, stream) {
+    const teed = stream.tee();
+    teed[1].readToEnd().then(result => {
+      console.log(str + ': ' + util.Uint8Array_to_str(result));
+    });
+    return teed[0];
+  },
+
   getLeftNBits: function (array, bitcount) {
     const rest = bitcount % 8;
     if (rest === 0) {

test/general/index.js

@@ -13,5 +13,6 @@ describe('General', function () {
   require('./x25519.js');
   require('./brainpool.js');
   require('./decompression.js');
+  require('./streaming.js');
 });
 

test/general/streaming.js

@@ -0,0 +1,60 @@
+const openpgp = typeof window !== 'undefined' && window.openpgp ? window.openpgp : require('../../dist/openpgp');
+
+const stub = require('sinon/lib/sinon/stub');
+const chai = require('chai');
+chai.use(require('chai-as-promised'));
+
+const { expect } = chai;
+
+const { Stream, util } = openpgp;
+
+describe('Streaming', function() {
+  it('Encrypt small message', async function() {
+    const data = new ReadableStream({
+      async start(controller) {
+        controller.enqueue(util.str_to_Uint8Array('hello '));
+        controller.enqueue(util.str_to_Uint8Array('world'));
+        controller.close();
+      }
+    });
+    const encrypted = await openpgp.encrypt({
+      data,
+      passwords: ['test'],
+    });
+    const msgAsciiArmored = util.Uint8Array_to_str(await encrypted.data.readToEnd());
+    const message = openpgp.message.readArmored(msgAsciiArmored);
+    const decrypted = await openpgp.decrypt({
+      passwords: ['test'],
+      message
+    });
+    expect(decrypted.data).to.equal('hello world');
+  });
+
+  it('Encrypt larger message', async function() {
+    let plaintext = [];
+    let i = 0;
+    const data = new ReadableStream({
+      async pull(controller) {
+        if (i++ < 10) {
+          let randomBytes = await openpgp.crypto.random.getRandomBytes(1024);
+          controller.enqueue(randomBytes);
+          plaintext.push(randomBytes);
+        } else {
+          controller.close();
+        }
+      }
+    });
+    const encrypted = await openpgp.encrypt({
+      data,
+      passwords: ['test'],
+    });
+    const msgAsciiArmored = util.Uint8Array_to_str(await encrypted.data.readToEnd());
+    const message = openpgp.message.readArmored(msgAsciiArmored);
+    const decrypted = await openpgp.decrypt({
+      passwords: ['test'],
+      message,
+      format: 'binary'
+    });
+    expect(decrypted.data).to.deep.equal(util.concatUint8Array(plaintext));
+  });
+});